diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 491db1f58f..347c44aedd 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2200,4 +2200,34 @@ dataset: - name: updated_at data_categories: [system.operations] - name: username - data_categories: [user.account.username] \ No newline at end of file + data_categories: [user.account.username] + - name: plus_consent_automation + fields: + - name: connection_config_id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] + - name: plus_consentable_item + fields: + - name: consent_automation_id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: external_id + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + - name: name + data_categories: [system.operations] + - name: notice_id + data_categories: [system.operations] + - name: parent_id + data_categories: [system.operations] + - name: type + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index de5dc5eddd..80e708ada7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The types of changes are: - Add "status" field to detection & discovery tables [#5141](https://github.com/ethyca/fides/pull/5141) - Added optional filters `exclude_saas_datasets` and `only_unlinked_datasets` to the list datasets endpoint [#5132](https://github.com/ethyca/fides/pull/5132) - Add new config options to support notice-only banner and modal [#5136](https://github.com/ethyca/fides/pull/5136) +- Added models to support bidirectional consent (Fides Plus feature) [#5118](https://github.com/ethyca/fides/pull/5118) ### Changed - Moving Privacy Center endpoint logging behind debug flag [#5103](https://github.com/ethyca/fides/pull/5103) diff --git a/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py b/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py index 43281c510e..6cf278f0bd 100644 --- a/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py +++ b/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py @@ -12,7 +12,6 @@ from loguru import logger from sqlalchemy import text from sqlalchemy.engine import Connection -from sqlalchemy.orm.session import Session from fides.api.alembic.migrations.helpers.fideslang_migration_functions import ( remove_conflicting_rule_targets, diff --git a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py new file mode 100644 index 0000000000..4fa082335f --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py @@ -0,0 +1,100 @@ +"""add consent automation tables + +Revision ID: d69cf8f82a58 +Revises: fc82ab64bd5e +Create Date: 2024-07-24 23:09:46.681097 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d69cf8f82a58" +down_revision = "fc82ab64bd5e" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "plus_consent_automation", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("connection_config_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["connection_config_id"], ["connectionconfig.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("connection_config_id"), + ) + op.create_index( + op.f("ix_plus_consent_automation_id"), + "plus_consent_automation", + ["id"], + unique=False, + ) + op.create_table( + "plus_consentable_item", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("consent_automation_id", sa.String(), nullable=False), + sa.Column("external_id", sa.String(), nullable=False), + sa.Column("parent_id", sa.String(), nullable=True), + sa.Column("notice_id", sa.String(), nullable=True), + sa.Column("type", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["consent_automation_id"], + ["plus_consent_automation.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["notice_id"], + ["privacynotice.id"], + ), + sa.ForeignKeyConstraint( + ["parent_id"], ["plus_consentable_item.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("consent_automation_id", "type", "external_id"), + ) + op.create_index( + op.f("ix_plus_consentable_item_id"), + "plus_consentable_item", + ["id"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_plus_consentable_item_id"), table_name="plus_consentable_item" + ) + op.drop_table("plus_consentable_item") + op.drop_index( + op.f("ix_plus_consent_automation_id"), table_name="plus_consent_automation" + ) + op.drop_table("plus_consent_automation") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 85ad17f4ec..8a52d33bea 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -7,6 +7,7 @@ from fides.api.models.authentication_request import AuthenticationRequest from fides.api.models.client import ClientDetail from fides.api.models.connectionconfig import ConnectionConfig +from fides.api.models.consent_automation import ConsentAutomation from fides.api.models.custom_asset import CustomAsset from fides.api.models.custom_connector_template import CustomConnectorTemplate from fides.api.models.datasetconfig import DatasetConfig diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 7c125c4907..6db18fe222 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -7,7 +7,7 @@ from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.ext.mutable import MutableDict -from sqlalchemy.orm import Session, relationship +from sqlalchemy.orm import RelationshipProperty, Session, relationship from sqlalchemy_utils.types.encrypted.encrypted_type import ( AesGcmEngine, StringEncryptedType, @@ -15,6 +15,7 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists from fides.api.db.base_class import Base, FidesBase, JSONTypeOverride +from fides.api.models.consent_automation import ConsentAutomation from fides.api.models.sql_models import System # type: ignore[attr-defined] from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.saas_config import SaaSConfig @@ -164,6 +165,10 @@ class ConnectionConfig(Base): system = relationship(System, back_populates="connection_configs", uselist=False) + consent_automation: RelationshipProperty[Optional[ConsentAutomation]] = ( + relationship(ConsentAutomation, cascade="all, delete-orphan") + ) + # Identifies the privacy actions needed from this connection by the associated system. enabled_actions = Column( ARRAY(Enum(ActionType, native_enum=False)), unique=False, nullable=True diff --git a/src/fides/api/models/consent_automation.py b/src/fides/api/models/consent_automation.py new file mode 100644 index 0000000000..4d6ec2a699 --- /dev/null +++ b/src/fides/api/models/consent_automation.py @@ -0,0 +1,168 @@ +from typing import Any, Dict, List, Optional + +from loguru import logger +from sqlalchemy import Column, ForeignKey, String, UniqueConstraint +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import RelationshipProperty, Session, relationship + +from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined] +from fides.api.models.privacy_notice import PrivacyNotice + + +class ConsentAutomation(Base): + @declared_attr + def __tablename__(self) -> str: + return "plus_consent_automation" + + connection_config_id = Column( + String, + ForeignKey("connectionconfig.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + consentable_items: "RelationshipProperty[List[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="consent_automation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + @classmethod + def create( + cls, db: Session, *, data: Dict[str, Any], check_name: bool = False + ) -> "ConsentAutomation": + consentable_items = data.pop("consentable_items", []) + consent_automation = super().create(db=db, data=data, check_name=check_name) + link_consentable_items_to_consent_automation( + db, consentable_items, consent_automation + ) + return consent_automation + + def update(self, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation": + consentable_items = data.pop("consentable_items", []) + super().update(db=db, data=data) + link_consentable_items_to_consent_automation(db, consentable_items, self) + return self + + @classmethod + def create_or_update(cls, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation": # type: ignore[override] + consent_automation = ConsentAutomation.filter( + db=db, + conditions=( + ConsentAutomation.connection_config_id == data["connection_config_id"] + ), + ).first() + + if consent_automation: + consent_automation.update(db=db, data=data) + else: + consent_automation = cls.create(db=db, data=data) + + return consent_automation + + +def link_consentable_items_to_consent_automation( + db: Session, + consentable_items: List[Dict[str, Any]], + consent_automation: ConsentAutomation, +) -> None: + """ + Takes a hierarchical list of consentable items and maps them to database items. + Attaches the database items to the consent automation. + """ + + existing_items = { + (item.type, str(item.external_id)): item + for item in consent_automation.consentable_items + } + + def process_items( + items_data: List[Dict[str, Any]], parent_id: str = None + ) -> List[ConsentableItem]: + processed_items = [] + for item_data in items_data: + external_id = str(item_data["id"]) + item_type = item_data["type"] + + key = (item_type, external_id) + item = existing_items.get(key) + + if item: + item.notice_id = item_data.get("notice_id") + else: + item = ConsentableItem( + external_id=external_id, + consent_automation_id=consent_automation.id, + parent_id=parent_id, + type=item_type, + name=item_data["name"], + notice_id=item_data.get("notice_id"), + ) + db.add(item) + # flush to the DB so we can get the auto-generated ID for this item + db.flush() + + processed_items.append(item) + + if "children" in item_data: + processed_items.extend( + process_items(item_data["children"], parent_id=item.id) + ) + + return processed_items + + try: + consent_automation.consentable_items = process_items(consentable_items) + db.commit() + except IntegrityError as exc: + logger.error("Error occurred while attempting to save consentable items", exc) + + db.refresh(consent_automation) + + +class ConsentableItem(Base): + @declared_attr + def __tablename__(self) -> str: + return "plus_consentable_item" + + id = Column( + String(255), primary_key=True, index=True, default=FidesBase.generate_uuid + ) + + consent_automation_id = Column( + String, + ForeignKey(ConsentAutomation.id_field_path, ondelete="CASCADE"), + nullable=False, + ) + external_id = Column( + String, + nullable=False, + ) + parent_id = Column( + String, + ForeignKey("plus_consentable_item.id", ondelete="CASCADE"), + nullable=True, + ) + notice_id = Column(String, ForeignKey(PrivacyNotice.id_field_path), nullable=True) + type = Column( + String, + nullable=False, + ) + name = Column(String, nullable=False) + consent_automation = relationship( + "ConsentAutomation", back_populates="consentable_items" + ) + children: "RelationshipProperty[List[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="parent", + cascade="all, delete-orphan", + passive_deletes=True, + ) + parent: "RelationshipProperty[Optional[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="children", + remote_side=[id], + ) + + __table_args__ = (UniqueConstraint("consent_automation_id", "type", "external_id"),) diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py new file mode 100644 index 0000000000..5cdd6d88f8 --- /dev/null +++ b/src/fides/api/schemas/consentable_item.py @@ -0,0 +1,76 @@ +from typing import List, Optional, Type + +from pydantic import BaseModel, Field + +from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel + + +class ConsentableItem(BaseModel): + """ + Schema to represent 3rd-party consentable items and privacy notice relationships. + """ + + id: str + type: str + name: str + notice_id: Optional[str] = None + children: List["ConsentableItem"] = Field(default_factory=list) + unmapped: Optional[bool] = False + + @classmethod + def from_orm( + cls: Type["ConsentableItem"], obj: ConsentableItemModel + ) -> "ConsentableItem": + item = cls( + id=obj.external_id, + type=obj.type, + name=obj.name, + notice_id=obj.notice_id, + ) + # recursively set children + item.children = [ + cls.from_orm(child) for child in getattr(obj, "children", []) # type: ignore[pydantic-orm] + ] + return item + + +def merge_consentable_items( + api_items: List[ConsentableItem], db_items: Optional[List[ConsentableItem]] = None +) -> List[ConsentableItem]: + """ + Recursively merges the lists of consentable items, setting the unmapped flag if the item is not in the database. + + WARNING: This is a destructive operation for the api_items parameter. + """ + + if db_items is None: + db_items = [] + + def merge_consentable_items_recursive( + source: ConsentableItem, target: Optional[ConsentableItem] + ) -> None: + if target is None: + source.unmapped = True + for child in source.children: + child.unmapped = True + return + + if source.id == target.id: + source.unmapped = False + source.notice_id = target.notice_id + target_children_map = {child.id: child for child in target.children} + + for child in source.children: + target_child = target_children_map.get(child.id) + if target_child is not None: + merge_consentable_items_recursive(child, target_child) + + # create a map of target items for efficient lookup + db_item_map = {item.id: item for item in db_items} + + # iterate through API items and merge + for api_item in api_items: + target_item = db_item_map.get(api_item.id) + merge_consentable_items_recursive(api_item, target_item) + + return api_items diff --git a/src/fides/api/schemas/saas/saas_config.py b/src/fides/api/schemas/saas/saas_config.py index d306e3775b..0749dadbff 100644 --- a/src/fides/api/schemas/saas/saas_config.py +++ b/src/fides/api/schemas/saas/saas_config.py @@ -18,6 +18,10 @@ from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.shared_schemas import HTTPMethod +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestOverrideFactory, + SaaSRequestType, +) class ParamValue(BaseModel): @@ -555,8 +559,16 @@ def supported_actions(self) -> List[ActionType]: ): supported_actions.append(ActionType.erasure) - # check for consent - if self.consent_requests: + # consent is supported if the SaaSConfig has consent_requests defined + # or if the SaaSConfig.type has an UPDATE_CONSENT function + # registered in the SaaSRequestOverrideFactory + if ( + self.consent_requests + or self.type + in SaaSRequestOverrideFactory.registry[ + SaaSRequestType.UPDATE_CONSENT + ].keys() + ): supported_actions.append(ActionType.consent) return supported_actions diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 8f214f6293..7028f31192 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -1,6 +1,6 @@ import json from json import JSONDecodeError -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Optional, Tuple, Union, cast import pydash from loguru import logger @@ -37,6 +37,7 @@ PostProcessorStrategy, ) from fides.api.service.saas_request.saas_request_override_factory import ( + RequestOverrideFunction, SaaSRequestOverrideFactory, SaaSRequestType, ) @@ -104,25 +105,28 @@ def query_config(self, node: ExecutionNode) -> SaaSQueryConfig: def get_client_config(self) -> ClientConfig: """Utility method for getting client config according to the current class state""" saas_config_client_config = self.saas_config.client_config - required_current_saas_request = self.current_saas_request - assert required_current_saas_request is not None - current_request_client_config = required_current_saas_request.client_config + current_saas_request = self.current_saas_request - return current_request_client_config or saas_config_client_config + if ( + current_saas_request is not None + and current_saas_request.client_config is not None + ): + return current_saas_request.client_config + + return saas_config_client_config def get_rate_limit_config(self) -> Optional[RateLimitConfig]: """Utility method for getting rate limit config according to the current class state""" saas_config_rate_limit_config = self.saas_config.rate_limit_config - required_current_saas_request = self.current_saas_request - assert required_current_saas_request is not None - current_request_rate_limit_config = ( - required_current_saas_request.rate_limit_config - ) + if self.current_saas_request is not None: + current_request_rate_limit_config = ( + self.current_saas_request.rate_limit_config + ) + if current_request_rate_limit_config is not None: + return current_request_rate_limit_config - return ( - current_request_rate_limit_config or saas_config_rate_limit_config or None - ) + return saas_config_rate_limit_config def set_privacy_request_state( self, @@ -752,7 +756,7 @@ def _invoke_test_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.TEST ) @@ -785,7 +789,7 @@ def _invoke_read_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.READ ) @@ -825,7 +829,7 @@ def _invoke_masking_request_override( Includes the necessary data preparations for override input and has error handling for uncaught exceptions coming out of the override """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType(query_config.action) ) @@ -868,7 +872,7 @@ def _invoke_consent_request_override( Invokes the appropriate user-defined SaaS request override for consent requests and performs error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType(query_config.action) ) diff --git a/src/fides/api/service/saas_request/saas_request_override_factory.py b/src/fides/api/service/saas_request/saas_request_override_factory.py index 01c9dbd152..bf21bf0abc 100644 --- a/src/fides/api/service/saas_request/saas_request_override_factory.py +++ b/src/fides/api/service/saas_request/saas_request_override_factory.py @@ -8,6 +8,7 @@ InvalidSaaSRequestOverrideException, NoSuchSaaSRequestOverrideException, ) +from fides.api.schemas.consentable_item import ConsentableItem from fides.api.util.collection_util import Row @@ -24,6 +25,14 @@ class SaaSRequestType(Enum): DELETE = "delete" OPT_IN = "opt_in" OPT_OUT = "opt_out" + GET_CONSENTABLE_ITEMS = "get_consentable_items" + UPDATE_CONSENT = "update_consent" + PROCESS_CONSENT_WEBHOOK = "process_consent_webhook" + + +RequestOverrideFunction = Callable[ + ..., Union[List[Row], List[ConsentableItem], int, bool, None] +] class SaaSRequestOverrideFactory: @@ -32,9 +41,7 @@ class SaaSRequestOverrideFactory: user-defined functions that act as overrides to SaaS request execution """ - registry: Dict[ - SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, bool, None]]] - ] = {} + registry: Dict[SaaSRequestType, Dict[str, RequestOverrideFunction]] = {} valid_overrides: Dict[SaaSRequestType, str] = {} # initialize each request type's inner dicts with an empty dict @@ -44,8 +51,8 @@ class SaaSRequestOverrideFactory: @classmethod def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ - [Callable[..., Union[List[Row], int, bool, None]]], - Callable[..., Union[List[Row], int, bool, None]], + [RequestOverrideFunction], + RequestOverrideFunction, ]: """ Decorator to register the custom-implemented SaaS request override @@ -60,8 +67,8 @@ def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ ) def wrapper( - override_function: Callable[..., Union[List[Row], int, bool, None]], - ) -> Callable[..., Union[List[Row], int, bool, None]]: + override_function: RequestOverrideFunction, + ) -> RequestOverrideFunction: for request_type in request_types: logger.debug( "Registering new SaaS request override function '{}' under name '{}' for SaaSRequestType {}", @@ -83,6 +90,12 @@ def wrapper( validate_update_override_function(override_function) elif request_type in (SaaSRequestType.OPT_IN, SaaSRequestType.OPT_OUT): validate_consent_override_function(override_function) + elif request_type == SaaSRequestType.GET_CONSENTABLE_ITEMS: + validate_get_consentable_item_function(override_function) + elif request_type == SaaSRequestType.UPDATE_CONSENT: + validate_update_consent_function(override_function) + elif request_type == SaaSRequestType.PROCESS_CONSENT_WEBHOOK: + validate_process_consent_webhook_function(override_function) else: raise ValueError( f"Invalid SaaSRequestType '{request_type}' provided for SaaS request override function" @@ -109,16 +122,16 @@ def wrapper( @classmethod def get_override( cls, override_function_name: str, request_type: SaaSRequestType - ) -> Callable[..., Union[List[Row], int, bool, None]]: + ) -> RequestOverrideFunction: """ Returns the request override function given the name. Raises NoSuchSaaSRequestOverrideException if the named override does not exist. """ try: - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( - cls.registry[request_type][override_function_name] - ) + override_function: RequestOverrideFunction = cls.registry[request_type][ + override_function_name + ] except KeyError: raise NoSuchSaaSRequestOverrideException( f"Custom SaaS override '{override_function_name}' does not exist. Valid custom SaaS override classes for SaaSRequestType {request_type} are [{cls.valid_overrides[request_type]}]" @@ -213,5 +226,17 @@ def validate_consent_override_function(f: Callable) -> None: ) +def validate_get_consentable_item_function(f: Callable) -> None: + pass + + +def validate_update_consent_function(f: Callable) -> None: + pass + + +def validate_process_consent_webhook_function(f: Callable) -> None: + pass + + # TODO: Avoid running this on import? register = SaaSRequestOverrideFactory.register diff --git a/tests/ops/models/test_consent_automation.py b/tests/ops/models/test_consent_automation.py new file mode 100644 index 0000000000..bd5ba122c2 --- /dev/null +++ b/tests/ops/models/test_consent_automation.py @@ -0,0 +1,241 @@ +from sqlalchemy import and_ +from sqlalchemy.orm import Session + +from fides.api.models.consent_automation import ConsentableItem, ConsentAutomation + + +class TestConsentAutomation: + + def test_create_consent_automation(self, db: Session, connection_config): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + + def test_update_consent_automation_add_consentable_items( + self, db: Session, connection_config, privacy_notice + ): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + assert consent_automation.consentable_items[0].notice_id is None + assert consent_automation.consentable_items[1].notice_id is None + + consentable_items[0]["notice_id"] = privacy_notice.id + consent_automation.update( + db=db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + + for consentable_item in consent_automation.consentable_items: + if consentable_item.name == "Marketing channel (email)": + assert consentable_item.notice_id == privacy_notice.id + if consentable_item.name == "Weekly Ads": + assert consentable_item.notice_id is None + + consent_automation.delete(db) + + def test_update_consent_automation_remove_consentable_items( + self, db: Session, connection_config + ): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + assert consent_automation.consentable_items[0].notice_id is None + assert consent_automation.consentable_items[1].notice_id is None + + consentable_items[0]["children"] = [] + consent_automation.update( + db=db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 1 + + consent_automation.delete(db) + + def test_consent_automation_delete(self, db: Session, connection_config): + """ + Verify the related consentable items are deleted when the consent automation is deleted. + """ + + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + consent_automation = ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 2 + + consent_automation.delete(db) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 0 + + def test_consentable_item_delete(self, db: Session, connection_config): + """ + Verify the child consentable items are deleted when the parent is deleted. + """ + + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + consent_automation = ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consentable_items = ( + db.query(ConsentableItem) + .filter( + and_( + ConsentableItem.consent_automation_id == consent_automation.id, + ConsentableItem.parent_id.is_(None), + ) + ) + .all() + ) + assert len(consentable_items) == 1 + + parent_consentable_item = consentable_items[0] + parent_consentable_item.delete(db) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 0 diff --git a/tests/ops/schemas/test_consentable_item_schema.py b/tests/ops/schemas/test_consentable_item_schema.py new file mode 100644 index 0000000000..825b14ca77 --- /dev/null +++ b/tests/ops/schemas/test_consentable_item_schema.py @@ -0,0 +1,207 @@ +from fides.api.schemas.consentable_item import ConsentableItem, merge_consentable_items + + +class TestConsentableItemUtils: + def test_merge_consentable_items_single_level(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ + ConsentableItem(type="MessageType", id="1", name="Welcome"), + ConsentableItem(type="MessageType", id="2", name="Promotional"), + ], + ) + ] + db_values = [ + ConsentableItem( + type="Channel", + id="2", + name="SMS", + children=[ + ConsentableItem(type="MessageType", id="1", name="Welcome"), + ConsentableItem(type="MessageType", id="2", name="Transactional"), + ], + ) + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ), + ConsentableItem( + id="2", + type="MessageType", + name="Promotional", + notice_id=None, + children=[], + unmapped=True, + ), + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_nested(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Marketing", + children=[ + ConsentableItem(type="MessageType", id="1", name="Newsletter"), + ConsentableItem(type="MessageType", id="3", name="Promotional"), + ], + ) + ] + db_values = [ + ConsentableItem( + type="Channel", + id="1", + notice_id="notice_123", + name="Marketing", + children=[ + ConsentableItem( + type="MessageType", + id="1", + name="Newsletter", + notice_id="notice_456", + ), + ConsentableItem(type="MessageType", id="3", name="Transactional"), + ], + ) + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Marketing", + notice_id="notice_123", + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Newsletter", + notice_id="notice_456", + children=[], + unmapped=False, + ), + ConsentableItem( + id="3", + type="MessageType", + name="Promotional", + notice_id=None, + children=[], + unmapped=False, + ), + ], + unmapped=False, + ) + ] + + def test_merge_consentable_items_with_empty_list(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ConsentableItem(type="MessageType", id="1", name="Welcome")], + ) + ] + assert merge_consentable_items(api_values, []) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ) + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_with_none_value(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ConsentableItem(type="MessageType", id="1", name="Welcome")], + ) + ] + assert merge_consentable_items(api_values, None) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ) + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_multiple_items(self): + api_values = [ + ConsentableItem(type="Channel", id="1", name="Email"), + ConsentableItem(type="Channel", id="2", name="SMS"), + ConsentableItem(type="Channel", id="3", name="Push"), + ] + db_values = [ + ConsentableItem(type="Channel", id="1", name="Email"), + ConsentableItem(type="Channel", id="3", name="Push"), + ConsentableItem(type="Channel", id="4", name="Web"), + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[], + unmapped=False, + ), + ConsentableItem( + id="2", + type="Channel", + name="SMS", + notice_id=None, + children=[], + unmapped=True, + ), + ConsentableItem( + id="3", + type="Channel", + name="Push", + notice_id=None, + children=[], + unmapped=False, + ), + ]