diff --git a/Makefile b/Makefile index dd0e5acba..dea6d0edf 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ pylint: compose-build mypy: compose-build @echo "Running mypy checks..." @docker-compose run $(IMAGE_NAME) \ - mypy --ignore-missing-imports src/ + mypy src/ pytest: compose-build @echo "Running pytest unit tests..." diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 20a9be495..459c5a811 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "4a0d10fa-6f76-4395-8d9c-73dd49246e9f", + "_postman_id": "4cf21904-951b-45f2-9e0a-bab994b5d534", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, @@ -3321,6 +3321,67 @@ "response": [] } ] + }, + { + "name": "DRP", + "item": [ + { + "name": "Exercise", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"meta\": {\"version\": \"0.5\"},\n \"regime\": \"ccpa\",\n \"exercise\": [\n \"access\"\n ],\n \"identity\": \"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20ifQ.4I8XLWnTYp8oMHjN2ypP3Hpg45DIaGNAEmj1QCYONUI\",\n}" + }, + "url": { + "raw": "{{host}}/drp/exercise", + "host": [ + "{{host}}" + ], + "path": [ + "drp", + "exercise" + ] + } + }, + "response": [] + }, + { + "name": "Status", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/drp/status?request_id={{privacy_request_id}}", + "host": [ + "{{host}}" + ], + "path": [ + "drp", + "status" + ], + "query": [ + { + "key": "request_id", + "value": "{{privacy_request_id}}" + } + ] + } + }, + "response": [] + } + ] } ], "event": [ diff --git a/fidesops.toml b/fidesops.toml index 3f06eafbb..5fe6a6bc3 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -19,6 +19,7 @@ CORS_ORIGINS=["http://localhost", "http://localhost:8080", "http://localhost:300 ENCODING="UTF-8" OAUTH_ROOT_CLIENT_ID="fidesopsadmin" OAUTH_ROOT_CLIENT_SECRET="fidesopsadminsecret" +DRP_JWT_SECRET="secret" [execution] TASK_RETRY_COUNT=0 diff --git a/requirements.txt b/requirements.txt index 9e63cab5e..e690bbdd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,4 +31,5 @@ PyMySQL==1.0.2 sqlalchemy-redshift==0.8.8 snowflake-sqlalchemy==1.3.2 sqlalchemy-bigquery==1.3.0 -multidimensional_urlencode==0.0.4 \ No newline at end of file +multidimensional_urlencode==0.0.4 +pyjwt diff --git a/src/fidesops/api/v1/api.py b/src/fidesops/api/v1/api.py index c49130271..8e217dd3f 100644 --- a/src/fidesops/api/v1/api.py +++ b/src/fidesops/api/v1/api.py @@ -4,6 +4,7 @@ config_endpoints, connection_endpoints, dataset_endpoints, + drp_endpoints, encryption_endpoints, health_endpoints, masking_endpoints, @@ -22,6 +23,7 @@ api_router.include_router(config_endpoints.router) api_router.include_router(connection_endpoints.router) api_router.include_router(dataset_endpoints.router) +api_router.include_router(drp_endpoints.router) api_router.include_router(encryption_endpoints.router) api_router.include_router(health_endpoints.router) api_router.include_router(masking_endpoints.router) diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py new file mode 100644 index 000000000..392391354 --- /dev/null +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -0,0 +1,153 @@ +import logging +from typing import Dict, Any, Optional + +import jwt +from fastapi import HTTPException, Depends, APIRouter, Security +from sqlalchemy.orm import Session +from starlette.status import ( + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_424_FAILED_DEPENDENCY, + HTTP_200_OK, + HTTP_500_INTERNAL_SERVER_ERROR, +) + +from fidesops import common_exceptions +from fidesops.api import deps +from fidesops.api.v1 import scope_registry as scopes +from fidesops.api.v1 import urn_registry as urls +from fidesops.core.config import config +from fidesops.models.policy import Policy +from fidesops.models.privacy_request import PrivacyRequest +from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate, DrpIdentity +from fidesops.schemas.privacy_request import PrivacyRequestDRPStatusResponse +from fidesops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.service.drp.drp_fidesops_mapper import DrpFidesopsMapper +from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner +from fidesops.service.privacy_request.request_service import ( + build_required_privacy_request_kwargs, + cache_data, +) +from fidesops.util.cache import FidesopsRedis +from fidesops.util.oauth_util import verify_oauth_client + +logger = logging.getLogger(__name__) +router = APIRouter(tags=["DRP"], prefix=urls.V1_URL_PREFIX) +EMBEDDED_EXECUTION_LOG_LIMIT = 50 + + +@router.post( + urls.DRP_EXERCISE, + status_code=HTTP_200_OK, + response_model=PrivacyRequestDRPStatusResponse, +) +def create_drp_privacy_request( + *, + cache: FidesopsRedis = Depends(deps.get_cache), + db: Session = Depends(deps.get_db), + data: DrpPrivacyRequestCreate, +) -> PrivacyRequestDRPStatusResponse: + """ + Given a drp privacy request body, create and execute + a corresponding Fidesops PrivacyRequest + """ + + jwt_key: str = config.security.DRP_JWT_SECRET + if jwt_key is None: + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail="JWT key must be provided", + ) + + logger.info(f"Finding policy with drp action '{data.exercise[0]}'") + policy: Optional[Policy] = Policy.get_by( + db=db, + field="drp_action", + value=data.exercise[0], + ) + + if not policy: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No policy found with drp action '{data.exercise}'.", + ) + + privacy_request_kwargs: Dict[str, Any] = build_required_privacy_request_kwargs( + None, policy.id + ) + + try: + privacy_request: PrivacyRequest = PrivacyRequest.create( + db=db, data=privacy_request_kwargs + ) + + logger.info(f"Decrypting identity for DRP privacy request {privacy_request.id}") + + decrypted_identity: DrpIdentity = DrpIdentity( + **jwt.decode(data.identity, jwt_key, algorithms=["HS256"]) + ) + + mapped_identity: PrivacyRequestIdentity = DrpFidesopsMapper.map_identity( + drp_identity=decrypted_identity + ) + + cache_data(privacy_request, policy, mapped_identity, None, data) + + PrivacyRequestRunner( + cache=cache, + privacy_request=privacy_request, + ).submit() + + return PrivacyRequestDRPStatusResponse( + request_id=privacy_request.id, + received_at=privacy_request.requested_at, + status=DrpFidesopsMapper.map_status(privacy_request.status), + ) + + except common_exceptions.RedisConnectionError as exc: + logger.error("RedisConnectionError: %s", exc) + # Thrown when cache.ping() fails on cache connection retrieval + raise HTTPException( + status_code=HTTP_424_FAILED_DEPENDENCY, + detail=exc.args[0], + ) + except Exception as exc: + logger.error(f"Exception: {exc}") + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail="DRP privacy request could not be exercised", + ) + + +@router.get( + urls.DRP_STATUS, + dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], + response_model=PrivacyRequestDRPStatusResponse, +) +def get_request_status_drp( + *, db: Session = Depends(deps.get_db), request_id: str +) -> PrivacyRequestDRPStatusResponse: + """ + Returns PrivacyRequest information where the respective privacy request is associated with + a policy that implements a Data Rights Protocol action. + """ + + logger.info(f"Finding request for DRP with ID: {request_id}") + request = PrivacyRequest.get( + db=db, + id=request_id, + ) + if not request or not request.policy or not request.policy.drp_action: + # If no request is found with this ID, or that request has no policy, + # or that request's policy has no associated drp_action. + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"Privacy request with ID {request_id} does not exist, or is not associated with a data rights protocol action.", + ) + + logger.info(f"Privacy request with ID: {request_id} found for DRP status.") + return PrivacyRequestDRPStatusResponse( + request_id=request.id, + received_at=request.requested_at, + status=DrpFidesopsMapper.map_status(request.status), + ) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index 3af905d94..ae4eb43a9 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -3,7 +3,7 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, DefaultDict, Dict, List, Optional, Set, Union +from typing import Any, Callable, DefaultDict, Dict, List, Optional, Union from fastapi import APIRouter, Body, Depends, HTTPException, Security from fastapi_pagination import Page, Params @@ -56,8 +56,6 @@ PrivacyRequestVerboseResponse, ReviewPrivacyRequestIds, DenyPrivacyRequests, - PrivacyRequestDRPStatusResponse, - PrivacyRequestDRPStatus, ) from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner from fidesops.service.privacy_request.request_service import ( @@ -428,62 +426,6 @@ def get_request_status( return paginated -def _map_fidesops_status_to_drp_status( - status: PrivacyRequestStatus, -) -> PrivacyRequestDRPStatus: - PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING: Dict[ - PrivacyRequestStatus, PrivacyRequestDRPStatus - ] = { - PrivacyRequestStatus.pending: PrivacyRequestDRPStatus.open, - PrivacyRequestStatus.approved: PrivacyRequestDRPStatus.in_progress, - PrivacyRequestStatus.denied: PrivacyRequestDRPStatus.denied, - PrivacyRequestStatus.in_processing: PrivacyRequestDRPStatus.in_progress, - PrivacyRequestStatus.complete: PrivacyRequestDRPStatus.fulfilled, - PrivacyRequestStatus.paused: PrivacyRequestDRPStatus.in_progress, - PrivacyRequestStatus.error: PrivacyRequestDRPStatus.expired, - } - try: - return PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING[status] - except KeyError: - raise ValueError(f"Request has invalid DRP request status: {status.value}") - - -@router.get( - urls.REQUEST_STATUS_DRP, - dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], - response_model=PrivacyRequestDRPStatusResponse, -) -def get_request_status_drp( - privacy_request_id: str, - *, - db: Session = Depends(deps.get_db), -) -> PrivacyRequestDRPStatusResponse: - """ - Returns PrivacyRequest information where the respective privacy request is associated with - a policy that implements a Data Rights Protocol action. - """ - - logger.info(f"Finding request for DRP with ID: {privacy_request_id}") - request = PrivacyRequest.get( - db=db, - id=privacy_request_id, - ) - if not request or not request.policy or not request.policy.drp_action: - # If no request is found with this ID, or that request has no policy, - # or that request's policy has no associated drp_action. - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"Privacy request with ID {privacy_request_id} does not exist, or is not associated with a data rights protocol action.", - ) - - logger.info(f"Privacy request with ID: {privacy_request_id} found for DRP status.") - return PrivacyRequestDRPStatusResponse( - request_id=request.id, - received_at=request.requested_at, - status=_map_fidesops_status_to_drp_status(request.status), - ) - - @router.get( urls.REQUEST_STATUS_LOGS, dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], diff --git a/src/fidesops/api/v1/urn_registry.py b/src/fidesops/api/v1/urn_registry.py index 939fbd9e5..39442422d 100644 --- a/src/fidesops/api/v1/urn_registry.py +++ b/src/fidesops/api/v1/urn_registry.py @@ -37,7 +37,6 @@ REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log" PRIVACY_REQUEST_RESUME = "/privacy-request/{privacy_request_id}/resume" REQUEST_PREVIEW = "/privacy-request/preview" -REQUEST_STATUS_DRP = "/privacy-request/{privacy_request_id}/drp" # Rule URLs RULE_LIST = "/policy/{policy_key}/rule" @@ -88,3 +87,7 @@ # Health URL HEALTH = "/health" + +# DRP +DRP_EXERCISE = "/drp/exercise" +DRP_STATUS = "/drp/status" diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 7dab15186..001c88843 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -120,6 +120,7 @@ class SecuritySettings(FidesSettings): AES_ENCRYPTION_KEY_LENGTH: int = 16 AES_GCM_NONCE_LENGTH: int = 12 APP_ENCRYPTION_KEY: str + DRP_JWT_SECRET: str @validator("APP_ENCRYPTION_KEY") def validate_encryption_key_length( diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 4f21a0074..deaf1ff75 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -186,10 +186,17 @@ def cache_drp_request_body(self, drp_request_body: DrpPrivacyRequestCreate) -> N drp_request_body_dict: Dict[str, Any] = dict(drp_request_body) for key, value in drp_request_body_dict.items(): if value is not None: - cache.set_with_autoexpire( - get_drp_request_body_cache_key(self.id, key), - value, - ) + # handle nested dict/objects + if not isinstance(value, (bytes, str, int, float)): + cache.set_with_autoexpire( + get_drp_request_body_cache_key(self.id, key), + repr(value), + ) + else: + cache.set_with_autoexpire( + get_drp_request_body_cache_key(self.id, key), + value, + ) def cache_encryption(self, encryption_key: Optional[str] = None) -> None: """Sets the encryption key in the Fidesops app cache if provided""" diff --git a/src/fidesops/schemas/drp_privacy_request.py b/src/fidesops/schemas/drp_privacy_request.py index 784df605a..6d0e1015e 100644 --- a/src/fidesops/schemas/drp_privacy_request.py +++ b/src/fidesops/schemas/drp_privacy_request.py @@ -1,6 +1,8 @@ from enum import Enum from typing import Optional, List +from pydantic import validator + from fidesops.models.policy import DrpAction from fidesops.schemas.base_class import BaseSchema @@ -22,7 +24,7 @@ class DrpPrivacyRequestCreate(BaseSchema): meta: DrpMeta regime: Optional[DrpRegime] - exercise: DrpAction + exercise: List[DrpAction] relationships: Optional[List[str]] identity: str status_callback: Optional[str] @@ -31,3 +33,25 @@ class Config: """Populate models with the raw value of enum fields, rather than the enum itself""" use_enum_values = True + + @validator("exercise") + def check_exercise_length(cls, exercise: [List[DrpAction]]) -> List[DrpAction]: + """Validate the only one exercise action is provided""" + if len(exercise) > 1: + raise ValueError("Multiple exercise actions are not supported at this time") + return exercise + + +class DrpIdentity(BaseSchema): + """Drp identity props""" + + aud: Optional[str] + sub: Optional[str] + name: Optional[str] + email: Optional[str] + email_verified: Optional[bool] + phone_number: Optional[str] + phone_number_verified: Optional[bool] + address: Optional[str] + address_verified: Optional[bool] + owner_of_attorney: Optional[str] diff --git a/src/fidesops/service/drp/drp_fidesops_mapper.py b/src/fidesops/service/drp/drp_fidesops_mapper.py new file mode 100644 index 000000000..16b45c599 --- /dev/null +++ b/src/fidesops/service/drp/drp_fidesops_mapper.py @@ -0,0 +1,57 @@ +import logging +from typing import Dict + +from fidesops.models.privacy_request import PrivacyRequestStatus +from fidesops.schemas.drp_privacy_request import DrpIdentity +from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus +from fidesops.schemas.redis_cache import PrivacyRequestIdentity + +logger = logging.getLogger(__name__) + + +class DrpFidesopsMapper: + """ + Map DRP objects/enums to Fidesops + """ + + @staticmethod + def map_identity(drp_identity: DrpIdentity) -> PrivacyRequestIdentity: + """ + Currently, both email and phone_number identity props map 1:1 to the corresponding + Fidesops identity props in PrivacyRequestIdentity. This may not always be the case. + This class also allows us to implement custom logic to handle "verified" id props. + """ + fidesops_identity_kwargs: Dict[str, str] = {} + DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP: Dict[str, str] = { + "email": "email", + "phone_number": "phone_number", + } + for attr, val in drp_identity.__dict__.items(): + if attr not in DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP.keys(): + logger.warning( + f"Identity attribute of {attr} is not supported by Fidesops at this time. Continuing to use other identity props, if provided." + ) + else: + fidesops_prop: str = DRP_TO_FIDESOPS_SUPPORTED_IDENTITY_PROPS_MAP[attr] + fidesops_identity_kwargs[fidesops_prop] = val + return PrivacyRequestIdentity(**fidesops_identity_kwargs) + + @staticmethod + def map_status( + status: PrivacyRequestStatus, + ) -> PrivacyRequestDRPStatus: + PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING: Dict[ + PrivacyRequestStatus, PrivacyRequestDRPStatus + ] = { + PrivacyRequestStatus.pending: PrivacyRequestDRPStatus.open, + PrivacyRequestStatus.approved: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.denied: PrivacyRequestDRPStatus.denied, + PrivacyRequestStatus.in_processing: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.complete: PrivacyRequestDRPStatus.fulfilled, + PrivacyRequestStatus.paused: PrivacyRequestDRPStatus.in_progress, + PrivacyRequestStatus.error: PrivacyRequestDRPStatus.expired, + } + try: + return PRIVACY_REQUEST_STATUS_TO_DRP_MAPPING[status] + except KeyError: + raise ValueError(f"Request has invalid DRP request status: {status.value}") diff --git a/tests/api/v1/endpoints/test_drp_endpoints.py b/tests/api/v1/endpoints/test_drp_endpoints.py new file mode 100644 index 000000000..f1da6d01c --- /dev/null +++ b/tests/api/v1/endpoints/test_drp_endpoints.py @@ -0,0 +1,350 @@ +from typing import Callable +from unittest import mock + +import jwt +import pytest +from sqlalchemy.orm import Session +from starlette.testclient import TestClient + +from fidesops.api.v1.scope_registry import ( + PRIVACY_REQUEST_READ, + STORAGE_CREATE_OR_UPDATE, +) +from fidesops.api.v1.urn_registry import ( + V1_URL_PREFIX, + DRP_EXERCISE, + DRP_STATUS, +) +from fidesops.core.config import config + +from fidesops.models.privacy_request import ( + PrivacyRequest, + PrivacyRequestStatus, +) +from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus +from fidesops.util.cache import get_drp_request_body_cache_key, get_identity_cache_key + + +class TestCreateDrpPrivacyRequest: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + DRP_EXERCISE + + @mock.patch( + "fidesops.service.privacy_request.request_runner_service.PrivacyRequestRunner.submit" + ) + def test_create_drp_privacy_request( + self, + run_access_request_mock, + url, + db, + api_client: TestClient, + policy_drp_action, + cache, + ): + + identity = {"email": "test@example.com"} + encoded_identity: str = jwt.encode( + identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + ) + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": ["access"], + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 200 + response_data = resp.json() + assert response_data["status"] == "open" + assert response_data["received_at"] + assert response_data["request_id"] + pr = PrivacyRequest.get(db=db, id=response_data["request_id"]) + + # test appropriate data is cached + meta_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="meta", + ) + assert cache.get(meta_key) == "DrpMeta(version='0.5')" + regime_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="regime", + ) + assert cache.get(regime_key) == "ccpa" + exercise_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="exercise", + ) + assert cache.get(exercise_key) == "['access']" + identity_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="identity", + ) + assert ( + cache.get(identity_key) + == "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20ifQ.4I8XLWnTYp8oMHjN2ypP3Hpg45DIaGNAEmj1QCYONUI" + ) + fidesops_identity_key = get_identity_cache_key( + privacy_request_id=pr.id, + identity_attribute="email", + ) + assert cache.get(fidesops_identity_key) == identity["email"] + pr.delete(db=db) + assert run_access_request_mock.called + + @mock.patch( + "fidesops.service.privacy_request.request_runner_service.PrivacyRequestRunner.submit" + ) + def test_create_drp_privacy_request_unsupported_identity_props( + self, + run_access_request_mock, + url, + db, + api_client: TestClient, + policy_drp_action, + cache, + ): + + identity = {"email": "test@example.com", "address": "something"} + encoded_identity: str = jwt.encode( + identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + ) + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": ["access"], + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 200 + response_data = resp.json() + assert response_data["status"] == "open" + assert response_data["received_at"] + assert response_data["request_id"] + pr = PrivacyRequest.get(db=db, id=response_data["request_id"]) + + # test appropriate data is cached + meta_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="meta", + ) + assert cache.get(meta_key) == "DrpMeta(version='0.5')" + regime_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="regime", + ) + assert cache.get(regime_key) == "ccpa" + exercise_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="exercise", + ) + assert cache.get(exercise_key) == "['access']" + identity_key = get_drp_request_body_cache_key( + privacy_request_id=pr.id, + identity_attribute="identity", + ) + assert ( + cache.get(identity_key) + == "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20iLCJhZGRyZXNzIjoic29tZXRoaW5nIn0.VhHzwTNoTjuny7lSebD6_hc0SU8kEZDr3YegONMMfmY" + ) + fidesops_identity_key = get_identity_cache_key( + privacy_request_id=pr.id, + identity_attribute="email", + ) + assert cache.get(fidesops_identity_key) == identity["email"] + fidesops_identity_key_address = get_identity_cache_key( + privacy_request_id=pr.id, + identity_attribute="address", + ) + assert cache.get(fidesops_identity_key_address) is None + pr.delete(db=db) + assert run_access_request_mock.called + + def test_create_drp_privacy_request_no_jwt( + self, + url, + db, + api_client: TestClient, + policy_drp_action, + ): + + original_secret = config.security.DRP_JWT_SECRET + config.security.DRP_JWT_SECRET = None + identity = {"email": "test@example.com"} + encoded_identity: str = jwt.encode(identity, "secret", algorithm="HS256") + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": ["access"], + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 500 + config.security.DRP_JWT_SECRET = original_secret + + def test_create_drp_privacy_request_no_exercise( + self, + url, + db, + api_client: TestClient, + policy_drp_action, + ): + + identity = {"email": "test@example.com"} + encoded_identity: str = jwt.encode( + identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + ) + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": None, + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 422 + + def test_create_drp_privacy_request_invalid_exercise( + self, + url, + db, + api_client: TestClient, + policy_drp_action, + ): + + identity = {"email": "test@example.com"} + encoded_identity: str = jwt.encode( + identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + ) + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": ["access", "deletion"], + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 422 + + def test_create_drp_privacy_request_no_associated_policy( + self, + url, + db, + api_client: TestClient, + policy, + ): + + identity = {"email": "test@example.com"} + encoded_identity: str = jwt.encode( + identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + ) + data = { + "meta": {"version": "0.5"}, + "regime": "ccpa", + "exercise": ["access"], + "identity": encoded_identity, + } + resp = api_client.post(url, json=data) + assert resp.status_code == 404 + + +class TestGetPrivacyRequestDRP: + """ + Tests for the endpoint retrieving privacy requests specific to the DRP. + """ + + @pytest.fixture(scope="function") + def url_for_privacy_request( + self, + privacy_request: PrivacyRequest, + ) -> str: + return V1_URL_PREFIX + DRP_STATUS + f"?request_id={privacy_request.id}" + + @pytest.fixture(scope="function") + def url_for_privacy_request_with_drp_action( + self, + privacy_request_with_drp_action: PrivacyRequest, + ) -> str: + return ( + V1_URL_PREFIX + + DRP_STATUS + + f"?request_id={privacy_request_with_drp_action.id}" + ) + + def test_get_privacy_requests_unauthenticated( + self, + api_client: TestClient, + url_for_privacy_request: str, + ): + response = api_client.get( + url_for_privacy_request, + headers={}, + ) + assert 401 == response.status_code + + def test_get_privacy_requests_wrong_scope( + self, + api_client: TestClient, + generate_auth_header: Callable, + url_for_privacy_request: str, + ): + auth_header = generate_auth_header(scopes=[STORAGE_CREATE_OR_UPDATE]) + response = api_client.get( + url_for_privacy_request, + headers=auth_header, + ) + assert 403 == response.status_code + + def test_get_non_drp_privacy_request( + self, + api_client: TestClient, + generate_auth_header: Callable, + url_for_privacy_request: str, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get( + url_for_privacy_request, + headers=auth_header, + ) + assert 404 == response.status_code + privacy_request_id = url_for_privacy_request.split("=")[-1] + assert ( + response.json()["detail"] + == f"Privacy request with ID {privacy_request_id} does not exist, or is not associated with a data rights protocol action." + ) + + @pytest.mark.parametrize( + "privacy_request_status,expected_drp_status", + [ + (PrivacyRequestStatus.pending, PrivacyRequestDRPStatus.open), + (PrivacyRequestStatus.approved, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.denied, PrivacyRequestDRPStatus.denied), + (PrivacyRequestStatus.in_processing, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.complete, PrivacyRequestDRPStatus.fulfilled), + (PrivacyRequestStatus.paused, PrivacyRequestDRPStatus.in_progress), + (PrivacyRequestStatus.error, PrivacyRequestDRPStatus.expired), + ], + ) + def test_get_privacy_request_with_drp_action( + self, + api_client: TestClient, + db: Session, + generate_auth_header: Callable, + url_for_privacy_request_with_drp_action: str, + privacy_request_with_drp_action: PrivacyRequest, + privacy_request_status: PrivacyRequestStatus, + expected_drp_status: PrivacyRequestDRPStatus, + ): + privacy_request_with_drp_action.status = privacy_request_status + privacy_request_with_drp_action.save(db=db) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get( + url_for_privacy_request_with_drp_action, + headers=auth_header, + ) + assert 200 == response.status_code + assert expected_drp_status.value == response.json()["status"] + assert privacy_request_with_drp_action.id == response.json()["request_id"] + assert ( + privacy_request_with_drp_action.requested_at.isoformat() + == response.json()["received_at"] + ) diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index d68bd0292..796985277 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -6,13 +6,12 @@ import json from datetime import datetime from dateutil.parser import parse -from typing import Callable, List +from typing import List from unittest import mock from fastapi_pagination import Params import pytest from starlette.testclient import TestClient -from sqlalchemy.orm import Session from fidesops.api.v1.endpoints.privacy_request_endpoints import ( EMBEDDED_EXECUTION_LOG_LIMIT, @@ -25,7 +24,6 @@ DATASETS, PRIVACY_REQUEST_APPROVE, PRIVACY_REQUEST_DENY, - REQUEST_STATUS_DRP, ) from fidesops.api.v1.scope_registry import ( STORAGE_CREATE_OR_UPDATE, @@ -51,7 +49,6 @@ JWE_ISSUED_AT, ) from fidesops.schemas.masking.masking_secrets import SecretType -from fidesops.schemas.privacy_request import PrivacyRequestDRPStatus from fidesops.util.cache import ( get_identity_cache_key, get_encryption_cache_key, @@ -377,109 +374,6 @@ def test_create_privacy_request_no_identities( assert len(response_data) == 1 -class TestGetPrivacyRequestDRP: - """ - Tests for the endpoint retrieving privacy requests specific to the DRP. - """ - - @pytest.fixture(scope="function") - def url_for_privacy_request( - self, - privacy_request: PrivacyRequest, - ) -> str: - return V1_URL_PREFIX + REQUEST_STATUS_DRP.format( - privacy_request_id=privacy_request.id - ) - - @pytest.fixture(scope="function") - def url_for_privacy_request_with_drp_action( - self, - privacy_request_with_drp_action: PrivacyRequest, - ) -> str: - return V1_URL_PREFIX + REQUEST_STATUS_DRP.format( - privacy_request_id=privacy_request_with_drp_action.id - ) - - def test_get_privacy_requests_unauthenticated( - self, - api_client: TestClient, - url_for_privacy_request: str, - ): - response = api_client.get( - url_for_privacy_request, - headers={}, - ) - assert 401 == response.status_code - - def test_get_privacy_requests_wrong_scope( - self, - api_client: TestClient, - generate_auth_header: Callable, - url_for_privacy_request: str, - ): - auth_header = generate_auth_header(scopes=[STORAGE_CREATE_OR_UPDATE]) - response = api_client.get( - url_for_privacy_request, - headers=auth_header, - ) - assert 403 == response.status_code - - def test_get_non_drp_privacy_request( - self, - api_client: TestClient, - generate_auth_header: Callable, - url_for_privacy_request: str, - ): - auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) - response = api_client.get( - url_for_privacy_request, - headers=auth_header, - ) - assert 404 == response.status_code - privacy_request_id = url_for_privacy_request.split("/")[-2] - assert ( - response.json()["detail"] - == f"Privacy request with ID {privacy_request_id} does not exist, or is not associated with a data rights protocol action." - ) - - @pytest.mark.parametrize( - "privacy_request_status,expected_drp_status", - [ - (PrivacyRequestStatus.pending, PrivacyRequestDRPStatus.open), - (PrivacyRequestStatus.approved, PrivacyRequestDRPStatus.in_progress), - (PrivacyRequestStatus.denied, PrivacyRequestDRPStatus.denied), - (PrivacyRequestStatus.in_processing, PrivacyRequestDRPStatus.in_progress), - (PrivacyRequestStatus.complete, PrivacyRequestDRPStatus.fulfilled), - (PrivacyRequestStatus.paused, PrivacyRequestDRPStatus.in_progress), - (PrivacyRequestStatus.error, PrivacyRequestDRPStatus.expired), - ], - ) - def test_get_privacy_request_with_drp_action( - self, - api_client: TestClient, - db: Session, - generate_auth_header: Callable, - url_for_privacy_request_with_drp_action: str, - privacy_request_with_drp_action: PrivacyRequest, - privacy_request_status: PrivacyRequestStatus, - expected_drp_status: PrivacyRequestDRPStatus, - ): - privacy_request_with_drp_action.status = privacy_request_status - privacy_request_with_drp_action.save(db=db) - auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) - response = api_client.get( - url_for_privacy_request_with_drp_action, - headers=auth_header, - ) - assert 200 == response.status_code - assert expected_drp_status.value == response.json()["status"] - assert privacy_request_with_drp_action.id == response.json()["request_id"] - assert ( - privacy_request_with_drp_action.requested_at.isoformat() - == response.json()["received_at"] - ) - - class TestGetPrivacyRequests: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail) -> str: diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 2260d2bfc..144006651 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -146,7 +146,9 @@ def hubspot_erasure_data( retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is False: if not retries: - raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot") + raise Exception( + f"Contact with contact id {contact_id} could not be added to Hubspot" + ) retries -= 1 time.sleep(5) @@ -163,7 +165,9 @@ def hubspot_erasure_data( retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is True: if not retries: - raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot") + raise Exception( + f"Contact with contact id {contact_id} could not be deleted from Hubspot" + ) retries -= 1 time.sleep(5) # Ensures contact is deleted diff --git a/tests/integration_tests/saas/test_segment_task.py b/tests/integration_tests/saas/test_segment_task.py index c00a6135e..60bc335d8 100644 --- a/tests/integration_tests/saas/test_segment_task.py +++ b/tests/integration_tests/saas/test_segment_task.py @@ -143,7 +143,7 @@ def test_segment_saas_erasure_request_task( segment_connection_config, segment_dataset_config, segment_erasure_identity_email, - segment_erasure_data + segment_erasure_data, ) -> None: """Full erasure request based on the Segment SaaS config""" config.execution.MASKING_STRICT = False # Allow GDPR Delete