From ca5f1138b128d56a7e7fa15c771ef97909686507 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 22 Jul 2024 10:53:52 -0700 Subject: [PATCH 01/11] Changes to support bidirectional consent --- .../api/service/connectors/saas_connector.py | 25 ++++++++++--------- .../saas_request_override_factory.py | 21 ++++++++++++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 8f214f6293..1830869838 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -104,25 +104,26 @@ def query_config(self, node: ExecutionNode) -> SaaSQueryConfig: def get_client_config(self) -> ClientConfig: """Utility method for getting client config according to the current class state""" saas_config_client_config = self.saas_config.client_config - required_current_saas_request = self.current_saas_request - assert required_current_saas_request is not None - current_request_client_config = required_current_saas_request.client_config + current_saas_request = self.current_saas_request - return current_request_client_config or saas_config_client_config + if ( + current_saas_request is not None + and current_saas_request.client_config is not None + ): + return current_saas_request.client_config + + return saas_config_client_config def get_rate_limit_config(self) -> Optional[RateLimitConfig]: """Utility method for getting rate limit config according to the current class state""" saas_config_rate_limit_config = self.saas_config.rate_limit_config - required_current_saas_request = self.current_saas_request - assert required_current_saas_request is not None - current_request_rate_limit_config = ( - required_current_saas_request.rate_limit_config - ) + if self.current_saas_request is not None: + current_request_rate_limit_config = self.current_saas_request.rate_limit_config + if current_request_rate_limit_config is not None: + return current_request_rate_limit_config - return ( - current_request_rate_limit_config or saas_config_rate_limit_config or None - ) + return saas_config_rate_limit_config def set_privacy_request_state( self, diff --git a/src/fides/api/service/saas_request/saas_request_override_factory.py b/src/fides/api/service/saas_request/saas_request_override_factory.py index 01c9dbd152..cbea5b6bc8 100644 --- a/src/fides/api/service/saas_request/saas_request_override_factory.py +++ b/src/fides/api/service/saas_request/saas_request_override_factory.py @@ -24,6 +24,9 @@ class SaaSRequestType(Enum): DELETE = "delete" OPT_IN = "opt_in" OPT_OUT = "opt_out" + GET_CONSENTABLE_ITEMS = "get_consentable_items" + UPDATE_CONSENT = "update_consent" + PROCESS_CONSENT_WEBHOOK = "process_consent_webhook" class SaaSRequestOverrideFactory: @@ -83,6 +86,12 @@ def wrapper( validate_update_override_function(override_function) elif request_type in (SaaSRequestType.OPT_IN, SaaSRequestType.OPT_OUT): validate_consent_override_function(override_function) + elif request_type == SaaSRequestType.GET_CONSENTABLE_ITEMS: + validate_get_consentable_item_function(override_function) + elif request_type == SaaSRequestType.UPDATE_CONSENT: + validate_update_consent_function(override_function) + elif request_type == SaaSRequestType.PROCESS_CONSENT_WEBHOOK: + validate_process_consent_webhook_function(override_function) else: raise ValueError( f"Invalid SaaSRequestType '{request_type}' provided for SaaS request override function" @@ -213,5 +222,17 @@ def validate_consent_override_function(f: Callable) -> None: ) +def validate_get_consentable_item_function(f: Callable) -> None: + pass + + +def validate_update_consent_function(f: Callable) -> None: + pass + + +def validate_process_consent_webhook_function(f: Callable) -> None: + pass + + # TODO: Avoid running this on import? register = SaaSRequestOverrideFactory.register From 45ea7911743b592042e841fa7396ac9001cd9876 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 22 Jul 2024 15:11:09 -0700 Subject: [PATCH 02/11] Adding consent automation table --- ...c55911216f_add_consent_automation_table.py | 57 +++++++++++++++++++ src/fides/api/db/base.py | 1 + src/fides/api/models/connectionconfig.py | 4 ++ src/fides/api/models/consent_automation.py | 20 +++++++ tests/ops/models/test_consent_automation.py | 36 ++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py create mode 100644 src/fides/api/models/consent_automation.py create mode 100644 tests/ops/models/test_consent_automation.py diff --git a/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py b/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py new file mode 100644 index 0000000000..eb5d3316c6 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py @@ -0,0 +1,57 @@ +"""add consent automation table + +Revision ID: 17c55911216f +Revises: f712aa9429f4 +Create Date: 2024-07-22 21:36:45.948148 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "17c55911216f" +down_revision = "f712aa9429f4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "plus_consent_automation", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("connection_config_id", sa.String(), nullable=False), + sa.Column( + "consentable_items", postgresql.JSONB(astext_type=sa.Text()), nullable=False + ), + sa.ForeignKeyConstraint( + ["connection_config_id"], ["connectionconfig.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_plus_consent_automation_id"), + "plus_consent_automation", + ["id"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_plus_consent_automation_id"), table_name="plus_consent_automation" + ) + op.drop_table("plus_consent_automation") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 85ad17f4ec..8a52d33bea 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -7,6 +7,7 @@ from fides.api.models.authentication_request import AuthenticationRequest from fides.api.models.client import ClientDetail from fides.api.models.connectionconfig import ConnectionConfig +from fides.api.models.consent_automation import ConsentAutomation from fides.api.models.custom_asset import CustomAsset from fides.api.models.custom_connector_template import CustomConnectorTemplate from fides.api.models.datasetconfig import DatasetConfig diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 7c125c4907..1424228470 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -164,6 +164,10 @@ class ConnectionConfig(Base): system = relationship(System, back_populates="connection_configs", uselist=False) + consent_automations = relationship( + "ConsentAutomation", cascade="all, delete-orphan" + ) + # Identifies the privacy actions needed from this connection by the associated system. enabled_actions = Column( ARRAY(Enum(ActionType, native_enum=False)), unique=False, nullable=True diff --git a/src/fides/api/models/consent_automation.py b/src/fides/api/models/consent_automation.py new file mode 100644 index 0000000000..dab909979c --- /dev/null +++ b/src/fides/api/models/consent_automation.py @@ -0,0 +1,20 @@ +from sqlalchemy import Column, ForeignKey, String +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.mutable import MutableList + +from fides.api.db.base_class import Base # type: ignore[attr-defined] +from fides.api.models.connectionconfig import ConnectionConfig + + +class ConsentAutomation(Base): + @declared_attr + def __tablename__(self) -> str: + return "plus_consent_automation" + + connection_config_id = Column( + String, + ForeignKey(ConnectionConfig.id_field_path, ondelete="CASCADE"), + nullable=False, + ) + consentable_items = Column(MutableList.as_mutable(JSONB), nullable=False) diff --git a/tests/ops/models/test_consent_automation.py b/tests/ops/models/test_consent_automation.py new file mode 100644 index 0000000000..08ce182278 --- /dev/null +++ b/tests/ops/models/test_consent_automation.py @@ -0,0 +1,36 @@ +from sqlalchemy.orm import Session + +from fides.api.models.consent_automation import ConsentAutomation + + +class TestConsentAutomation: + def test_consentable_items(self, db: Session, connection_config): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "notice_id": "not_e0a4dbd1-7a66-4a5d-9ce8-f2d3125f84c8", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert consent_automation.consentable_items == consentable_items From c6817de519ede7d2961455d2626e8d194bed8701 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 24 Jul 2024 16:49:11 -0700 Subject: [PATCH 03/11] Refactoring consentable items to use a dedicated database table --- ...c55911216f_add_consent_automation_table.py | 57 ------- ...cf8f82a58_add_consent_automation_tables.py | 100 +++++++++++++ src/fides/api/models/connectionconfig.py | 7 +- src/fides/api/models/consent_automation.py | 139 +++++++++++++++++- .../api/service/connectors/saas_connector.py | 4 +- tests/ops/models/test_consent_automation.py | 116 ++++++++++++++- 6 files changed, 352 insertions(+), 71 deletions(-) delete mode 100644 src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py create mode 100644 src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py diff --git a/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py b/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py deleted file mode 100644 index eb5d3316c6..0000000000 --- a/src/fides/api/alembic/migrations/versions/17c55911216f_add_consent_automation_table.py +++ /dev/null @@ -1,57 +0,0 @@ -"""add consent automation table - -Revision ID: 17c55911216f -Revises: f712aa9429f4 -Create Date: 2024-07-22 21:36:45.948148 - -""" - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "17c55911216f" -down_revision = "f712aa9429f4" -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - "plus_consent_automation", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column("connection_config_id", sa.String(), nullable=False), - sa.Column( - "consentable_items", postgresql.JSONB(astext_type=sa.Text()), nullable=False - ), - sa.ForeignKeyConstraint( - ["connection_config_id"], ["connectionconfig.id"], ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index( - op.f("ix_plus_consent_automation_id"), - "plus_consent_automation", - ["id"], - unique=False, - ) - - -def downgrade(): - op.drop_index( - op.f("ix_plus_consent_automation_id"), table_name="plus_consent_automation" - ) - op.drop_table("plus_consent_automation") diff --git a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py new file mode 100644 index 0000000000..2a7bf15ae7 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py @@ -0,0 +1,100 @@ +"""add consent automation tables + +Revision ID: d69cf8f82a58 +Revises: f712aa9429f4 +Create Date: 2024-07-24 23:09:46.681097 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d69cf8f82a58" +down_revision = "f712aa9429f4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "plus_consent_automation", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("connection_config_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["connection_config_id"], ["connectionconfig.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("connection_config_id"), + ) + op.create_index( + op.f("ix_plus_consent_automation_id"), + "plus_consent_automation", + ["id"], + unique=False, + ) + op.create_table( + "plus_consentable_item", + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("consent_automation_id", sa.String(), nullable=False), + sa.Column("external_id", sa.String(), nullable=False), + sa.Column("parent_id", sa.String(), nullable=True), + sa.Column("notice_id", sa.String(), nullable=True), + sa.Column("type", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["consent_automation_id"], + ["plus_consent_automation.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["notice_id"], + ["privacynotice.id"], + ), + sa.ForeignKeyConstraint( + ["parent_id"], ["plus_consentable_item.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("consent_automation_id", "type", "external_id"), + ) + op.create_index( + op.f("ix_plus_consentable_item_id"), + "plus_consentable_item", + ["id"], + unique=False, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_plus_consentable_item_id"), table_name="plus_consentable_item" + ) + op.drop_table("plus_consentable_item") + op.drop_index( + op.f("ix_plus_consent_automation_id"), table_name="plus_consent_automation" + ) + op.drop_table("plus_consent_automation") diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 1424228470..6db18fe222 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -7,7 +7,7 @@ from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.ext.mutable import MutableDict -from sqlalchemy.orm import Session, relationship +from sqlalchemy.orm import RelationshipProperty, Session, relationship from sqlalchemy_utils.types.encrypted.encrypted_type import ( AesGcmEngine, StringEncryptedType, @@ -15,6 +15,7 @@ from fides.api.common_exceptions import KeyOrNameAlreadyExists from fides.api.db.base_class import Base, FidesBase, JSONTypeOverride +from fides.api.models.consent_automation import ConsentAutomation from fides.api.models.sql_models import System # type: ignore[attr-defined] from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.saas_config import SaaSConfig @@ -164,8 +165,8 @@ class ConnectionConfig(Base): system = relationship(System, back_populates="connection_configs", uselist=False) - consent_automations = relationship( - "ConsentAutomation", cascade="all, delete-orphan" + consent_automation: RelationshipProperty[Optional[ConsentAutomation]] = ( + relationship(ConsentAutomation, cascade="all, delete-orphan") ) # Identifies the privacy actions needed from this connection by the associated system. diff --git a/src/fides/api/models/consent_automation.py b/src/fides/api/models/consent_automation.py index dab909979c..a02bc05ae3 100644 --- a/src/fides/api/models/consent_automation.py +++ b/src/fides/api/models/consent_automation.py @@ -1,10 +1,13 @@ -from sqlalchemy import Column, ForeignKey, String -from sqlalchemy.dialects.postgresql import JSONB +from typing import Any, Dict, List, Optional + +from loguru import logger +from sqlalchemy import Column, ForeignKey, String, UniqueConstraint +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.ext.mutable import MutableList +from sqlalchemy.orm import RelationshipProperty, Session, relationship -from fides.api.db.base_class import Base # type: ignore[attr-defined] -from fides.api.models.connectionconfig import ConnectionConfig +from fides.api.db.base_class import Base, FidesBase # type: ignore[attr-defined] +from fides.api.models.privacy_notice import PrivacyNotice class ConsentAutomation(Base): @@ -14,7 +17,129 @@ def __tablename__(self) -> str: connection_config_id = Column( String, - ForeignKey(ConnectionConfig.id_field_path, ondelete="CASCADE"), + ForeignKey("connectionconfig.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + consentable_items: "RelationshipProperty[List[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="consent_automation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + + @classmethod + def create( + cls, db: Session, *, data: Dict[str, Any], check_name: bool = False + ) -> "ConsentAutomation": + consentable_items = data.pop("consentable_items", []) + consent_automation = super().create(db=db, data=data, check_name=check_name) + link_consentable_items_to_consent_automation( + db, consentable_items, consent_automation + ) + return consent_automation + + def update(self, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation": + consentable_items = data.pop("consentable_items", []) + super().update(db=db, data=data) + link_consentable_items_to_consent_automation(db, consentable_items, self) + return self + + +def link_consentable_items_to_consent_automation( + db: Session, + consentable_items: List[Dict[str, Any]], + consent_automation: ConsentAutomation, +) -> None: + existing_items = { + (item.type, str(item.external_id)): item + for item in consent_automation.consentable_items + } + + def process_items( + items_data: List[Dict[str, Any]], parent_id: str = None + ) -> List[ConsentableItem]: + processed_items = [] + for item_data in items_data: + external_id = str(item_data["id"]) + item_type = item_data["type"] + + key = (item_type, external_id) + item = existing_items.get(key) + + if item: + item.notice_id = item_data.get("notice_id") + else: + item = ConsentableItem( + external_id=external_id, + consent_automation_id=consent_automation.id, + parent_id=parent_id, + type=item_type, + name=item_data["name"], + notice_id=item_data.get("notice_id"), + ) + db.add(item) + + processed_items.append(item) + + if "children" in item_data: + processed_items.extend( + process_items(item_data["children"], parent_id=item.id) + ) + + return processed_items + + try: + consent_automation.consentable_items = process_items(consentable_items) + db.commit() + except IntegrityError as exc: + logger.error("Error occurred while attempting to save consentable items", exc) + + db.refresh(consent_automation) + + +class ConsentableItem(Base): + @declared_attr + def __tablename__(self) -> str: + return "plus_consentable_item" + + id = Column( + String(255), primary_key=True, index=True, default=FidesBase.generate_uuid + ) + + consent_automation_id = Column( + String, + ForeignKey(ConsentAutomation.id_field_path, ondelete="CASCADE"), + nullable=False, + ) + external_id = Column( + String, nullable=False, ) - consentable_items = Column(MutableList.as_mutable(JSONB), nullable=False) + parent_id = Column( + String, + ForeignKey("plus_consentable_item.id", ondelete="CASCADE"), + nullable=True, + ) + notice_id = Column(String, ForeignKey(PrivacyNotice.id_field_path), nullable=True) + type = Column( + String, + nullable=False, + ) + name = Column(String, nullable=False) + consent_automation = relationship( + "ConsentAutomation", back_populates="consentable_items" + ) + children: "RelationshipProperty[List[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="parent", + cascade="all, delete-orphan", + passive_deletes=True, + ) + parent: "RelationshipProperty[Optional[ConsentableItem]]" = relationship( + "ConsentableItem", + back_populates="children", + remote_side=[id], + ) + + __table_args__ = (UniqueConstraint("consent_automation_id", "type", "external_id"),) diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 1830869838..98877d1ffd 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -119,7 +119,9 @@ def get_rate_limit_config(self) -> Optional[RateLimitConfig]: saas_config_rate_limit_config = self.saas_config.rate_limit_config if self.current_saas_request is not None: - current_request_rate_limit_config = self.current_saas_request.rate_limit_config + current_request_rate_limit_config = ( + self.current_saas_request.rate_limit_config + ) if current_request_rate_limit_config is not None: return current_request_rate_limit_config diff --git a/tests/ops/models/test_consent_automation.py b/tests/ops/models/test_consent_automation.py index 08ce182278..9c845980b7 100644 --- a/tests/ops/models/test_consent_automation.py +++ b/tests/ops/models/test_consent_automation.py @@ -4,13 +4,103 @@ class TestConsentAutomation: - def test_consentable_items(self, db: Session, connection_config): + + def test_create_consent_automation(self, db: Session, connection_config): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + + def test_update_consent_automation_add_consentable_items( + self, db: Session, connection_config, privacy_notice + ): + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + assert consent_automation.consentable_items[0].notice_id is None + assert consent_automation.consentable_items[1].notice_id is None + + consentable_items[0]["notice_id"] = privacy_notice.id + consent_automation.update( + db=db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 2 + + for consentable_item in consent_automation.consentable_items: + if consentable_item.name == "Marketing channel (email)": + assert consentable_item.notice_id == privacy_notice.id + if consentable_item.name == "Weekly Ads": + assert consentable_item.notice_id is None + + consent_automation.delete(db) + + def test_update_consent_automation_remove_consentable_items( + self, db: Session, connection_config + ): consentable_items = [ { "type": "Channel", "id": 1, "name": "Marketing channel (email)", - "notice_id": "not_e0a4dbd1-7a66-4a5d-9ce8-f2d3125f84c8", "children": [ { "type": "Message type", @@ -33,4 +123,24 @@ def test_consentable_items(self, db: Session, connection_config): ) assert consent_automation is not None assert consent_automation.connection_config_id == connection_config.id - assert consent_automation.consentable_items == consentable_items + assert len(consent_automation.consentable_items) == 2 + assert consent_automation.consentable_items[0].notice_id is None + assert consent_automation.consentable_items[1].notice_id is None + + consentable_items[0]["children"] = [] + consent_automation.update( + db=db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consent_automation = ConsentAutomation.get_by( + db, field="connection_config_id", value=connection_config.id + ) + assert consent_automation is not None + assert consent_automation.connection_config_id == connection_config.id + assert len(consent_automation.consentable_items) == 1 + + consent_automation.delete(db) From 78b7da012cce173bc5871cbc37db74535c17a7e7 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 25 Jul 2024 14:05:03 -0700 Subject: [PATCH 04/11] Updating create or update for consentable items --- src/fides/api/models/consent_automation.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/fides/api/models/consent_automation.py b/src/fides/api/models/consent_automation.py index a02bc05ae3..4d6ec2a699 100644 --- a/src/fides/api/models/consent_automation.py +++ b/src/fides/api/models/consent_automation.py @@ -45,12 +45,33 @@ def update(self, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation": link_consentable_items_to_consent_automation(db, consentable_items, self) return self + @classmethod + def create_or_update(cls, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation": # type: ignore[override] + consent_automation = ConsentAutomation.filter( + db=db, + conditions=( + ConsentAutomation.connection_config_id == data["connection_config_id"] + ), + ).first() + + if consent_automation: + consent_automation.update(db=db, data=data) + else: + consent_automation = cls.create(db=db, data=data) + + return consent_automation + def link_consentable_items_to_consent_automation( db: Session, consentable_items: List[Dict[str, Any]], consent_automation: ConsentAutomation, ) -> None: + """ + Takes a hierarchical list of consentable items and maps them to database items. + Attaches the database items to the consent automation. + """ + existing_items = { (item.type, str(item.external_id)): item for item in consent_automation.consentable_items @@ -79,6 +100,8 @@ def process_items( notice_id=item_data.get("notice_id"), ) db.add(item) + # flush to the DB so we can get the auto-generated ID for this item + db.flush() processed_items.append(item) From f93333a40046dc6965847a31881d3ad8fd79c995 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 26 Jul 2024 18:30:13 -0700 Subject: [PATCH 05/11] Moving ConsentableItem schema to Fides --- src/fides/api/schemas/consentable_item.py | 75 +++++++ .../api/service/connectors/saas_connector.py | 11 +- .../saas_request_override_factory.py | 26 ++- .../schemas/test_consentable_item_schema.py | 207 ++++++++++++++++++ 4 files changed, 303 insertions(+), 16 deletions(-) create mode 100644 src/fides/api/schemas/consentable_item.py create mode 100644 tests/ops/schemas/test_consentable_item_schema.py diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py new file mode 100644 index 0000000000..3e63111290 --- /dev/null +++ b/src/fides/api/schemas/consentable_item.py @@ -0,0 +1,75 @@ +from typing import List, Optional, Type + +from pydantic import Field + +from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel +from fides.api.schemas.base_class import FidesSchema + + +class ConsentableItem(FidesSchema): + """ + Schema to represent 3rd-party consentable items and privacy notice relationships. + """ + + id: str + type: str + name: str + notice_id: Optional[str] = None + children: List["ConsentableItem"] = Field(default_factory=list) + unmapped: Optional[bool] = True + + @classmethod + def from_orm( + cls: Type["ConsentableItem"], obj: ConsentableItemModel + ) -> "ConsentableItem": + item = cls( + id=obj.external_id, + type=obj.type, + name=obj.name, + notice_id=obj.notice_id, + ) + # recursively set children + item.children = [cls.from_orm(child) for child in getattr(obj, "children", [])] + return item + + +def merge_consentable_items( + api_items: List[ConsentableItem], db_items: Optional[List[ConsentableItem]] = None +) -> List[ConsentableItem]: + """ + Recursively merges the lists of consentable items, setting the unmapped flag if the item is not in the database. + + WARNING: This is a destructive operation for the api_items parameter. + """ + + if db_items is None: + db_items = [] + + def merge_consentable_items_recursive( + source: ConsentableItem, target: Optional[ConsentableItem] + ) -> None: + if target is None: + source.unmapped = True + for child in source.children: + child.unmapped = True + return + + if source.id == target.id: + source.unmapped = False + source.notice_id = target.notice_id + target_children_map = {child.id: child for child in target.children} + + for child in source.children: + target_child = target_children_map.get(child.id) + if target_child is not None: + merge_consentable_items_recursive(child, target_child) + + # create a map of target items for efficient lookup + db_item_map = {item.id: item for item in db_items} + + # iterate through API items and merge + for api_item in api_items: + target_item = db_item_map.get(api_item.id) + merge_consentable_items_recursive(api_item, target_item) + + return api_items diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 98877d1ffd..7028f31192 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -1,6 +1,6 @@ import json from json import JSONDecodeError -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Optional, Tuple, Union, cast import pydash from loguru import logger @@ -37,6 +37,7 @@ PostProcessorStrategy, ) from fides.api.service.saas_request.saas_request_override_factory import ( + RequestOverrideFunction, SaaSRequestOverrideFactory, SaaSRequestType, ) @@ -755,7 +756,7 @@ def _invoke_test_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.TEST ) @@ -788,7 +789,7 @@ def _invoke_read_request_override( Contains error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType.READ ) @@ -828,7 +829,7 @@ def _invoke_masking_request_override( Includes the necessary data preparations for override input and has error handling for uncaught exceptions coming out of the override """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType(query_config.action) ) @@ -871,7 +872,7 @@ def _invoke_consent_request_override( Invokes the appropriate user-defined SaaS request override for consent requests and performs error handling for uncaught exceptions coming out of the override. """ - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( + override_function: RequestOverrideFunction = ( SaaSRequestOverrideFactory.get_override( override_function_name, SaaSRequestType(query_config.action) ) diff --git a/src/fides/api/service/saas_request/saas_request_override_factory.py b/src/fides/api/service/saas_request/saas_request_override_factory.py index cbea5b6bc8..bf21bf0abc 100644 --- a/src/fides/api/service/saas_request/saas_request_override_factory.py +++ b/src/fides/api/service/saas_request/saas_request_override_factory.py @@ -8,6 +8,7 @@ InvalidSaaSRequestOverrideException, NoSuchSaaSRequestOverrideException, ) +from fides.api.schemas.consentable_item import ConsentableItem from fides.api.util.collection_util import Row @@ -29,15 +30,18 @@ class SaaSRequestType(Enum): PROCESS_CONSENT_WEBHOOK = "process_consent_webhook" +RequestOverrideFunction = Callable[ + ..., Union[List[Row], List[ConsentableItem], int, bool, None] +] + + class SaaSRequestOverrideFactory: """ Factory class responsible for registering, maintaining, and providing user-defined functions that act as overrides to SaaS request execution """ - registry: Dict[ - SaaSRequestType, Dict[str, Callable[..., Union[List[Row], int, bool, None]]] - ] = {} + registry: Dict[SaaSRequestType, Dict[str, RequestOverrideFunction]] = {} valid_overrides: Dict[SaaSRequestType, str] = {} # initialize each request type's inner dicts with an empty dict @@ -47,8 +51,8 @@ class SaaSRequestOverrideFactory: @classmethod def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ - [Callable[..., Union[List[Row], int, bool, None]]], - Callable[..., Union[List[Row], int, bool, None]], + [RequestOverrideFunction], + RequestOverrideFunction, ]: """ Decorator to register the custom-implemented SaaS request override @@ -63,8 +67,8 @@ def register(cls, name: str, request_types: List[SaaSRequestType]) -> Callable[ ) def wrapper( - override_function: Callable[..., Union[List[Row], int, bool, None]], - ) -> Callable[..., Union[List[Row], int, bool, None]]: + override_function: RequestOverrideFunction, + ) -> RequestOverrideFunction: for request_type in request_types: logger.debug( "Registering new SaaS request override function '{}' under name '{}' for SaaSRequestType {}", @@ -118,16 +122,16 @@ def wrapper( @classmethod def get_override( cls, override_function_name: str, request_type: SaaSRequestType - ) -> Callable[..., Union[List[Row], int, bool, None]]: + ) -> RequestOverrideFunction: """ Returns the request override function given the name. Raises NoSuchSaaSRequestOverrideException if the named override does not exist. """ try: - override_function: Callable[..., Union[List[Row], int, bool, None]] = ( - cls.registry[request_type][override_function_name] - ) + override_function: RequestOverrideFunction = cls.registry[request_type][ + override_function_name + ] except KeyError: raise NoSuchSaaSRequestOverrideException( f"Custom SaaS override '{override_function_name}' does not exist. Valid custom SaaS override classes for SaaSRequestType {request_type} are [{cls.valid_overrides[request_type]}]" diff --git a/tests/ops/schemas/test_consentable_item_schema.py b/tests/ops/schemas/test_consentable_item_schema.py new file mode 100644 index 0000000000..825b14ca77 --- /dev/null +++ b/tests/ops/schemas/test_consentable_item_schema.py @@ -0,0 +1,207 @@ +from fides.api.schemas.consentable_item import ConsentableItem, merge_consentable_items + + +class TestConsentableItemUtils: + def test_merge_consentable_items_single_level(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ + ConsentableItem(type="MessageType", id="1", name="Welcome"), + ConsentableItem(type="MessageType", id="2", name="Promotional"), + ], + ) + ] + db_values = [ + ConsentableItem( + type="Channel", + id="2", + name="SMS", + children=[ + ConsentableItem(type="MessageType", id="1", name="Welcome"), + ConsentableItem(type="MessageType", id="2", name="Transactional"), + ], + ) + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ), + ConsentableItem( + id="2", + type="MessageType", + name="Promotional", + notice_id=None, + children=[], + unmapped=True, + ), + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_nested(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Marketing", + children=[ + ConsentableItem(type="MessageType", id="1", name="Newsletter"), + ConsentableItem(type="MessageType", id="3", name="Promotional"), + ], + ) + ] + db_values = [ + ConsentableItem( + type="Channel", + id="1", + notice_id="notice_123", + name="Marketing", + children=[ + ConsentableItem( + type="MessageType", + id="1", + name="Newsletter", + notice_id="notice_456", + ), + ConsentableItem(type="MessageType", id="3", name="Transactional"), + ], + ) + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Marketing", + notice_id="notice_123", + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Newsletter", + notice_id="notice_456", + children=[], + unmapped=False, + ), + ConsentableItem( + id="3", + type="MessageType", + name="Promotional", + notice_id=None, + children=[], + unmapped=False, + ), + ], + unmapped=False, + ) + ] + + def test_merge_consentable_items_with_empty_list(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ConsentableItem(type="MessageType", id="1", name="Welcome")], + ) + ] + assert merge_consentable_items(api_values, []) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ) + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_with_none_value(self): + api_values = [ + ConsentableItem( + type="Channel", + id="1", + name="Email", + children=[ConsentableItem(type="MessageType", id="1", name="Welcome")], + ) + ] + assert merge_consentable_items(api_values, None) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[ + ConsentableItem( + id="1", + type="MessageType", + name="Welcome", + notice_id=None, + children=[], + unmapped=True, + ) + ], + unmapped=True, + ) + ] + + def test_merge_consentable_items_multiple_items(self): + api_values = [ + ConsentableItem(type="Channel", id="1", name="Email"), + ConsentableItem(type="Channel", id="2", name="SMS"), + ConsentableItem(type="Channel", id="3", name="Push"), + ] + db_values = [ + ConsentableItem(type="Channel", id="1", name="Email"), + ConsentableItem(type="Channel", id="3", name="Push"), + ConsentableItem(type="Channel", id="4", name="Web"), + ] + assert merge_consentable_items(api_values, db_values) == [ + ConsentableItem( + id="1", + type="Channel", + name="Email", + notice_id=None, + children=[], + unmapped=False, + ), + ConsentableItem( + id="2", + type="Channel", + name="SMS", + notice_id=None, + children=[], + unmapped=True, + ), + ConsentableItem( + id="3", + type="Channel", + name="Push", + notice_id=None, + children=[], + unmapped=False, + ), + ] From 9abb34f3cad62a75da51be2f7514df4e0eb542e1 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 27 Jul 2024 17:14:08 -0700 Subject: [PATCH 06/11] Changing unmapped flag to default to false --- src/fides/api/schemas/consentable_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py index 3e63111290..f911b073e8 100644 --- a/src/fides/api/schemas/consentable_item.py +++ b/src/fides/api/schemas/consentable_item.py @@ -16,7 +16,7 @@ class ConsentableItem(FidesSchema): name: str notice_id: Optional[str] = None children: List["ConsentableItem"] = Field(default_factory=list) - unmapped: Optional[bool] = True + unmapped: Optional[bool] = False @classmethod def from_orm( From 1e26d2a7999106c291d18d569b96b48b8b50a500 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 29 Jul 2024 09:18:02 -0700 Subject: [PATCH 07/11] Updating supported actions logic --- src/fides/api/schemas/saas/saas_config.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/fides/api/schemas/saas/saas_config.py b/src/fides/api/schemas/saas/saas_config.py index d306e3775b..0749dadbff 100644 --- a/src/fides/api/schemas/saas/saas_config.py +++ b/src/fides/api/schemas/saas/saas_config.py @@ -18,6 +18,10 @@ from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.shared_schemas import HTTPMethod +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestOverrideFactory, + SaaSRequestType, +) class ParamValue(BaseModel): @@ -555,8 +559,16 @@ def supported_actions(self) -> List[ActionType]: ): supported_actions.append(ActionType.erasure) - # check for consent - if self.consent_requests: + # consent is supported if the SaaSConfig has consent_requests defined + # or if the SaaSConfig.type has an UPDATE_CONSENT function + # registered in the SaaSRequestOverrideFactory + if ( + self.consent_requests + or self.type + in SaaSRequestOverrideFactory.registry[ + SaaSRequestType.UPDATE_CONSENT + ].keys() + ): supported_actions.append(ActionType.consent) return supported_actions From fc764508e317a206d8a47e4fdf01bd78a034d3e0 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 29 Jul 2024 15:33:32 -0700 Subject: [PATCH 08/11] Misc fixes --- .fides/db_dataset.yml | 32 ++++++++++++++++++- ...cc7dc_migrate_remaining_data_categories.py | 1 - ...cf8f82a58_add_consent_automation_tables.py | 4 +-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 0764eee3b7..452d442563 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2188,4 +2188,34 @@ dataset: - name: updated_at data_categories: [system.operations] - name: username - data_categories: [user.account.username] \ No newline at end of file + data_categories: [user.account.username] + - name: plus_consent_automation + fields: + - name: connection_config_id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] + - name: plus_consentable_item + fields: + - name: consent_automation_id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: external_id + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + - name: name + data_categories: [system.operations] + - name: notice_id + data_categories: [system.operations] + - name: parent_id + data_categories: [system.operations] + - name: type + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] \ No newline at end of file diff --git a/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py b/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py index 43281c510e..6cf278f0bd 100644 --- a/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py +++ b/src/fides/api/alembic/migrations/versions/a6d9cdfcc7dc_migrate_remaining_data_categories.py @@ -12,7 +12,6 @@ from loguru import logger from sqlalchemy import text from sqlalchemy.engine import Connection -from sqlalchemy.orm.session import Session from fides.api.alembic.migrations.helpers.fideslang_migration_functions import ( remove_conflicting_rule_targets, diff --git a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py index 2a7bf15ae7..910cd46baa 100644 --- a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py +++ b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py @@ -1,7 +1,7 @@ """add consent automation tables Revision ID: d69cf8f82a58 -Revises: f712aa9429f4 +Revises: a6d9cdfcc7dc Create Date: 2024-07-24 23:09:46.681097 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "d69cf8f82a58" -down_revision = "f712aa9429f4" +down_revision = "a6d9cdfcc7dc" branch_labels = None depends_on = None From 61f9df4cb9085566592277a2fc0b29b2352e39dd Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 30 Jul 2024 09:38:33 -0700 Subject: [PATCH 09/11] Changing ConsentableItem to inherit from BaseModel instead of FidesSchema to prevent calling from_orm during payload validation --- src/fides/api/schemas/consentable_item.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py index f911b073e8..9732d948d0 100644 --- a/src/fides/api/schemas/consentable_item.py +++ b/src/fides/api/schemas/consentable_item.py @@ -1,12 +1,11 @@ from typing import List, Optional, Type -from pydantic import Field +from pydantic import BaseModel, Field from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel -from fides.api.schemas.base_class import FidesSchema -class ConsentableItem(FidesSchema): +class ConsentableItem(BaseModel): """ Schema to represent 3rd-party consentable items and privacy notice relationships. """ From 79fc16c17afb1468d161350a38c26eefa3188512 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 30 Jul 2024 14:29:42 -0700 Subject: [PATCH 10/11] Changes based on PR feedback --- tests/ops/models/test_consent_automation.py | 97 ++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/tests/ops/models/test_consent_automation.py b/tests/ops/models/test_consent_automation.py index 9c845980b7..bd5ba122c2 100644 --- a/tests/ops/models/test_consent_automation.py +++ b/tests/ops/models/test_consent_automation.py @@ -1,6 +1,7 @@ +from sqlalchemy import and_ from sqlalchemy.orm import Session -from fides.api.models.consent_automation import ConsentAutomation +from fides.api.models.consent_automation import ConsentableItem, ConsentAutomation class TestConsentAutomation: @@ -144,3 +145,97 @@ def test_update_consent_automation_remove_consentable_items( assert len(consent_automation.consentable_items) == 1 consent_automation.delete(db) + + def test_consent_automation_delete(self, db: Session, connection_config): + """ + Verify the related consentable items are deleted when the consent automation is deleted. + """ + + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + consent_automation = ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 2 + + consent_automation.delete(db) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 0 + + def test_consentable_item_delete(self, db: Session, connection_config): + """ + Verify the child consentable items are deleted when the parent is deleted. + """ + + consentable_items = [ + { + "type": "Channel", + "id": 1, + "name": "Marketing channel (email)", + "children": [ + { + "type": "Message type", + "id": 1, + "name": "Weekly Ads", + } + ], + } + ] + + consent_automation = ConsentAutomation.create_or_update( + db, + data={ + "connection_config_id": connection_config.id, + "consentable_items": consentable_items, + }, + ) + + consentable_items = ( + db.query(ConsentableItem) + .filter( + and_( + ConsentableItem.consent_automation_id == consent_automation.id, + ConsentableItem.parent_id.is_(None), + ) + ) + .all() + ) + assert len(consentable_items) == 1 + + parent_consentable_item = consentable_items[0] + parent_consentable_item.delete(db) + + consentable_items = ( + db.query(ConsentableItem) + .filter(ConsentableItem.consent_automation_id == consent_automation.id) + .all() + ) + assert len(consentable_items) == 0 From 3cf4ec6000b9aef2bfce3669c338313602a75d2d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 31 Jul 2024 09:35:49 -0700 Subject: [PATCH 11/11] Misc fixes and change log entry --- CHANGELOG.md | 1 + .../versions/d69cf8f82a58_add_consent_automation_tables.py | 4 ++-- src/fides/api/schemas/consentable_item.py | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 712f6c91a3..dc642d13e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The types of changes are: - Added support for displaying notices served in the Consent Banner [#5125](https://github.com/ethyca/fides/pull/5125) - Added ability to choose whether to use Opt In/Out buttons or Acknowledge button in the Consent Banner [#5125](https://github.com/ethyca/fides/pull/5125) - Add "status" field to detection & discovery tables [#5141](https://github.com/ethyca/fides/pull/5141) +- Added models to support bidirectional consent (Fides Plus feature) [#5118](https://github.com/ethyca/fides/pull/5118) ### Changed - Moving Privacy Center endpoint logging behind debug flag [#5103](https://github.com/ethyca/fides/pull/5103) diff --git a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py index 910cd46baa..4fa082335f 100644 --- a/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py +++ b/src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py @@ -1,7 +1,7 @@ """add consent automation tables Revision ID: d69cf8f82a58 -Revises: a6d9cdfcc7dc +Revises: fc82ab64bd5e Create Date: 2024-07-24 23:09:46.681097 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "d69cf8f82a58" -down_revision = "a6d9cdfcc7dc" +down_revision = "fc82ab64bd5e" branch_labels = None depends_on = None diff --git a/src/fides/api/schemas/consentable_item.py b/src/fides/api/schemas/consentable_item.py index 9732d948d0..5cdd6d88f8 100644 --- a/src/fides/api/schemas/consentable_item.py +++ b/src/fides/api/schemas/consentable_item.py @@ -28,7 +28,9 @@ def from_orm( notice_id=obj.notice_id, ) # recursively set children - item.children = [cls.from_orm(child) for child in getattr(obj, "children", [])] + item.children = [ + cls.from_orm(child) for child in getattr(obj, "children", []) # type: ignore[pydantic-orm] + ] return item