Updating privacy request search endpoint

CHANGELOG.md

@@ -26,6 +26,7 @@ The types of changes are:
 - Add an S3 connection type (currently used for discovery and detection only) [#4930](https://github.com/ethyca/fides/pull/4930)
 - Support for Limited FIDES__CELERY__* Env Vars [#4980](https://github.com/ethyca/fides/pull/4980)
 - Implement sending emails via property-specific messaging templates [#4950](https://github.com/ethyca/fides/pull/4950)
+- New privacy request search to replace existing endpoint [#4987](https://github.com/ethyca/fides/pull/4987)
 
 ### Changed
 - Move new data map reporting table out of beta and remove old table from Data Lineage map. [#4963](https://github.com/ethyca/fides/pull/4963)

src/fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -15,7 +15,7 @@
 from loguru import logger
 from pydantic import ValidationError as PydanticValidationError
 from pydantic import conlist
-from sqlalchemy import cast, column, null
+from sqlalchemy import cast, column, null, or_, select
 from sqlalchemy.orm import Query, Session
 from sqlalchemy.sql.expression import nullslast
 from starlette.responses import StreamingResponse
@@ -62,6 +62,7 @@
 from fides.api.models.privacy_request import (
     CheckpointActionRequired,
     ConsentRequest,
+    CustomPrivacyRequestField,
     ExecutionLog,
     ExecutionLogStatus,
     PrivacyRequest,
@@ -94,6 +95,7 @@
     ExecutionLogDetailResponse,
     ManualWebhookData,
     PrivacyRequestCreate,
+    PrivacyRequestFilter,
     PrivacyRequestNotificationInfo,
     PrivacyRequestResponse,
     PrivacyRequestTaskSchema,
@@ -153,6 +155,7 @@
     PRIVACY_REQUEST_RESUME,
     PRIVACY_REQUEST_RESUME_FROM_REQUIRES_INPUT,
     PRIVACY_REQUEST_RETRY,
+    PRIVACY_REQUEST_SEARCH,
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     PRIVACY_REQUEST_VERIFY_IDENTITY,
     PRIVACY_REQUESTS,
@@ -392,6 +395,8 @@ def _filter_privacy_request_queryset(
     query: Query,
     request_id: Optional[str] = None,
     identity: Optional[str] = None,
+    identities: Optional[Dict[str, Any]] = None,
+    custom_privacy_request_fields: Optional[Dict[str, Any]] = None,
     status: Optional[List[PrivacyRequestStatus]] = None,
     created_lt: Optional[datetime] = None,
     created_gt: Optional[datetime] = None,
@@ -408,8 +413,16 @@ def _filter_privacy_request_queryset(
     Utility method to apply filters to our privacy request query.
 
     Status supports "or" filtering:
-    ?status=approved&status=pending will be translated into an "or" query.
+    `status=["approved","pending"]` will be translated into an "or" query.
+
+    The `identities` and `custom_privacy_request_fields` parameters allow
+    searching for privacy requests that match any of the provided identities
+    or custom privacy request fields, respectively. The filtering is performed
+    using an "or" condition, meaning that a privacy request will be included
+    in the results if it matches at least one of the provided identities or
+    custom privacy request fields.
     """
+
     if any([completed_lt, completed_gt]) and any([errored_lt, errored_gt]):
         raise HTTPException(
             status_code=HTTP_400_BAD_REQUEST,
@@ -427,7 +440,7 @@ def _filter_privacy_request_queryset(
 
     if identity:
         hashed_identity = ProvidedIdentity.hash_value(value=identity)
-        identities: Set[str] = {
+        identity_set: Set[str] = {
             identity[0]
             for identity in ProvidedIdentity.filter(
                 db=db,
@@ -437,8 +450,40 @@ def _filter_privacy_request_queryset(
                 ),
             ).values(column("privacy_request_id"))
         }
-        query = query.filter(PrivacyRequest.id.in_(identities))
-    # Further restrict all PrivacyRequests by query params
+        query = query.filter(PrivacyRequest.id.in_(identity_set))
+
+    if identities:
+        identity_conditions = [
+            (ProvidedIdentity.field_name == field_name)
+            & (ProvidedIdentity.hashed_value == ProvidedIdentity.hash_value(value))
+            for field_name, value in identities.items()
+        ]
+
+        identities_query = select([ProvidedIdentity.privacy_request_id]).where(
+            or_(*identity_conditions)
+            & (ProvidedIdentity.privacy_request_id.isnot(None))
+        )
+        query = query.filter(PrivacyRequest.id.in_(identities_query))
+
+    if custom_privacy_request_fields:
+        custom_field_conditions = [
+            (CustomPrivacyRequestField.field_name == field_name)
+            & (
+                CustomPrivacyRequestField.hashed_value
+                == CustomPrivacyRequestField.hash_value(value)
+            )
+            for field_name, value in custom_privacy_request_fields.items()
+        ]
+
+        custom_fields_query = select(
+            [CustomPrivacyRequestField.privacy_request_id]
+        ).where(
+            or_(*custom_field_conditions)
+            & (CustomPrivacyRequestField.privacy_request_id.isnot(None))
+        )
+        query = query.filter(PrivacyRequest.id.in_(custom_fields_query))
+
+    # Further restrict all PrivacyRequests by additional params
     if request_id:
         query = query.filter(PrivacyRequest.id.ilike(f"{request_id}%"))
     if external_id:
@@ -535,25 +580,15 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None:
     )
 
 
-@router.get(
-    PRIVACY_REQUESTS,
-    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],
-    response_model=Page[
-        Union[
-            PrivacyRequestVerboseResponse,
-            PrivacyRequestResponse,
-        ]
-    ],
-)
-def get_request_status(
+def _shared_privacy_request_search(
     *,
-    db: Session = Depends(deps.get_db),
-    params: Params = Depends(),
+    db: Session,
+    params: Params,
     request_id: Optional[str] = None,
     identity: Optional[str] = None,
-    status: Optional[List[PrivacyRequestStatus]] = FastAPIQuery(
-        default=None
-    ),  # type:ignore
+    identities: Optional[Dict[str, str]] = None,
+    custom_privacy_request_fields: Optional[Dict[str, Any]] = None,
+    status: Optional[List[PrivacyRequestStatus]] = None,
     created_lt: Optional[datetime] = None,
     created_gt: Optional[datetime] = None,
     started_lt: Optional[datetime] = None,
@@ -571,11 +606,14 @@ def get_request_status(
     sort_field: str = "created_at",
     sort_direction: ColumnSort = ColumnSort.DESC,
 ) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]:
-    """Returns PrivacyRequest information. Supports a variety of optional query params.
+    """
+    Internal function to handle the logic for retrieving privacy requests.
 
-    To fetch a single privacy request, use the request_id query param `?request_id=`.
-    To see individual execution logs, use the verbose query param `?verbose=True`.
+    This function is used by both the GET and POST versions of the privacy request endpoints
+    to avoid duplicating the logic while transitioning from the GET version to the
+    POST version of the endpoint.
     """
+
     logger.info("Finding all request statuses with pagination params {}", params)
 
     query = db.query(PrivacyRequest)
@@ -584,6 +622,8 @@ def get_request_status(
         query,
         request_id,
         identity,
+        identities,
+        custom_privacy_request_fields,
         status,
         created_lt,
         created_gt,
@@ -634,6 +674,135 @@ def get_request_status(
     return paginated
 
 
+@router.get(
+    PRIVACY_REQUESTS,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],
+    response_model=Page[
+        Union[
+            PrivacyRequestVerboseResponse,
+            PrivacyRequestResponse,
+        ]
+    ],
+    deprecated=True,
+)
+def get_request_status(
+    *,
+    db: Session = Depends(deps.get_db),
+    params: Params = Depends(),
+    request_id: Optional[str] = None,
+    identity: Optional[str] = None,
+    status: Optional[List[PrivacyRequestStatus]] = FastAPIQuery(
+        default=None
+    ),  # type:ignore
+    created_lt: Optional[datetime] = None,
+    created_gt: Optional[datetime] = None,
+    started_lt: Optional[datetime] = None,
+    started_gt: Optional[datetime] = None,
+    completed_lt: Optional[datetime] = None,
+    completed_gt: Optional[datetime] = None,
+    errored_lt: Optional[datetime] = None,
+    errored_gt: Optional[datetime] = None,
+    external_id: Optional[str] = None,
+    action_type: Optional[ActionType] = None,
+    verbose: Optional[bool] = False,
+    include_identities: Optional[bool] = False,
+    include_custom_privacy_request_fields: Optional[bool] = False,
+    download_csv: Optional[bool] = False,
+    sort_field: str = "created_at",
+    sort_direction: ColumnSort = ColumnSort.DESC,
+) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]:
+    """
+    **This endpoint is deprecated. Please use `POST /privacy-request/search`,
+    which uses body parameters instead of query parameters for filtering.**
+
+    Returns PrivacyRequest information. Supports a variety of optional query params.
+
+    To fetch a single privacy request, use the request_id query param `?request_id=`.
+    To see individual execution logs, use the verbose query param `?verbose=True`.
+    """
+
+    # Both the old and new versions of the privacy request search endpoints use this shared function.
+    # The `identities` and `custom_privacy_request_fields` parameters are only supported in the new version
+    # so they are both set to None here.
+    return _shared_privacy_request_search(
+        db=db,
+        params=params,
+        request_id=request_id,
+        identity=identity,
+        identities=None,
+        custom_privacy_request_fields=None,
+        status=status,
+        created_lt=created_lt,
+        created_gt=created_gt,
+        started_lt=started_lt,
+        started_gt=started_gt,
+        completed_lt=completed_lt,
+        completed_gt=completed_gt,
+        errored_lt=errored_lt,
+        errored_gt=errored_gt,
+        external_id=external_id,
+        action_type=action_type,
+        verbose=verbose,
+        include_identities=include_identities,
+        include_custom_privacy_request_fields=include_custom_privacy_request_fields,
+        download_csv=download_csv,
+        sort_field=sort_field,
+        sort_direction=sort_direction,
+    )
+
+
+@router.post(
+    PRIVACY_REQUEST_SEARCH,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],
+    response_model=Page[
+        Union[
+            PrivacyRequestVerboseResponse,
+            PrivacyRequestResponse,
+        ]
+    ],
+)
+def privacy_request_search(
+    *,
+    db: Session = Depends(deps.get_db),
+    params: Params = Depends(),
+    privacy_request_filter: Optional[PrivacyRequestFilter] = Body(None),
+) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]:
+    """
+    Returns PrivacyRequest information. Supports a variety of optional filter parameters.
+
+    To see individual execution logs, set `"verbose": true`.
+    """
+
+    # default filter if the payload is empty
+    if privacy_request_filter is None:
+        privacy_request_filter = PrivacyRequestFilter()
+
+    return _shared_privacy_request_search(
+        db=db,
+        params=params,
+        request_id=privacy_request_filter.request_id,
+        identities=privacy_request_filter.identities,
+        custom_privacy_request_fields=privacy_request_filter.custom_privacy_request_fields,
+        status=privacy_request_filter.status,  # type: ignore
+        created_lt=privacy_request_filter.created_lt,
+        created_gt=privacy_request_filter.created_gt,
+        started_lt=privacy_request_filter.started_lt,
+        started_gt=privacy_request_filter.started_gt,
+        completed_lt=privacy_request_filter.completed_lt,
+        completed_gt=privacy_request_filter.completed_gt,
+        errored_lt=privacy_request_filter.errored_lt,
+        errored_gt=privacy_request_filter.errored_gt,
+        external_id=privacy_request_filter.external_id,
+        action_type=privacy_request_filter.action_type,
+        verbose=privacy_request_filter.verbose,
+        include_identities=privacy_request_filter.include_identities,
+        include_custom_privacy_request_fields=privacy_request_filter.include_custom_privacy_request_fields,
+        download_csv=privacy_request_filter.download_csv,
+        sort_field=privacy_request_filter.sort_field,
+        sort_direction=privacy_request_filter.sort_direction,
+    )
+
+
 @router.get(
     REQUEST_STATUS_LOGS,
     dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_READ])],

src/fides/api/schemas/privacy_request.py

@@ -3,7 +3,7 @@
 from typing import Any, Dict, List, Optional, Type, Union
 
 from fideslang.validation import FidesKey
-from pydantic import Field, validator
+from pydantic import Extra, Field, validator
 
 from fides.api.custom_types import SafeStr
 from fides.api.models.audit_log import AuditLogAction
@@ -20,6 +20,7 @@
 from fides.api.schemas.user import PrivacyRequestReviewer
 from fides.api.util.collection_util import Row
 from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key
+from fides.api.util.enums import ColumnSort
 from fides.config import CONFIG
 
 
@@ -339,3 +340,45 @@ class RequestTaskCallbackRequest(FidesSchema):
     rows_masked: Optional[int] = Field(
         default=None, description="Number of records masked, as an integer"
     )
+
+
+class PrivacyRequestFilter(FidesSchema):
+    request_id: Optional[str] = None
+    identities: Optional[Dict[str, Any]] = Field(
+        None, example={"email": "user@example.com", "loyalty_id": "CH-1"}
+    )
+    custom_privacy_request_fields: Optional[Dict[str, Any]] = Field(
+        None, example={"site_id": "abc", "subscriber_id": "123"}
+    )
+    status: Optional[Union[PrivacyRequestStatus, List[PrivacyRequestStatus]]] = None
+    created_lt: Optional[datetime] = None
+    created_gt: Optional[datetime] = None
+    started_lt: Optional[datetime] = None
+    started_gt: Optional[datetime] = None
+    completed_lt: Optional[datetime] = None
+    completed_gt: Optional[datetime] = None
+    errored_lt: Optional[datetime] = None
+    errored_gt: Optional[datetime] = None
+    external_id: Optional[str] = None
+    action_type: Optional[ActionType] = None
+    verbose: Optional[bool] = False
+    include_identities: Optional[bool] = False
+    include_custom_privacy_request_fields: Optional[bool] = False
+    download_csv: Optional[bool] = False
+    sort_field: str = "created_at"
+    sort_direction: ColumnSort = ColumnSort.DESC
+
+    class Config:
+        extra = Extra.forbid
+
+    @validator("status")
+    def validate_status_field(
+        cls,
+        field_value: Union[PrivacyRequestStatus, List[PrivacyRequestStatus]],
+    ) -> List[PrivacyRequestStatus]:
+        """
+        Keeps the status field flexible but converts a single value to a list for consistent processing.
+        """
+        if isinstance(field_value, PrivacyRequestStatus):
+            return [field_value]
+        return field_value

src/fides/common/api/v1/urn_registry.py

@@ -82,6 +82,7 @@
 PRIVACY_REQUEST_RESUME = "/privacy-request/{privacy_request_id}/resume"
 PRIVACY_REQUEST_NOTIFICATIONS = "/privacy-request/notification"
 PRIVACY_REQUEST_RETRY = "/privacy-request/{privacy_request_id}/retry"
+PRIVACY_REQUEST_SEARCH = "/privacy-request/search"
 REQUEST_PREVIEW = "/privacy-request/preview"
 PRIVACY_REQUEST_MANUAL_WEBHOOK_ACCESS_INPUT = (
     "/privacy-request/{privacy_request_id}/access_manual_webhook/{connection_key}"