Changes to support bidirectional consent

src/fides/api/alembic/migrations/versions/d69cf8f82a58_add_consent_automation_tables.py

@@ -0,0 +1,100 @@
+"""add consent automation tables
+
+Revision ID: d69cf8f82a58
+Revises: f712aa9429f4
+Create Date: 2024-07-24 23:09:46.681097
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d69cf8f82a58"
+down_revision = "f712aa9429f4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "plus_consent_automation",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("connection_config_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connection_config_id"], ["connectionconfig.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("connection_config_id"),
+    )
+    op.create_index(
+        op.f("ix_plus_consent_automation_id"),
+        "plus_consent_automation",
+        ["id"],
+        unique=False,
+    )
+    op.create_table(
+        "plus_consentable_item",
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column("consent_automation_id", sa.String(), nullable=False),
+        sa.Column("external_id", sa.String(), nullable=False),
+        sa.Column("parent_id", sa.String(), nullable=True),
+        sa.Column("notice_id", sa.String(), nullable=True),
+        sa.Column("type", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["consent_automation_id"],
+            ["plus_consent_automation.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["notice_id"],
+            ["privacynotice.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent_id"], ["plus_consentable_item.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("consent_automation_id", "type", "external_id"),
+    )
+    op.create_index(
+        op.f("ix_plus_consentable_item_id"),
+        "plus_consentable_item",
+        ["id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        op.f("ix_plus_consentable_item_id"), table_name="plus_consentable_item"
+    )
+    op.drop_table("plus_consentable_item")
+    op.drop_index(
+        op.f("ix_plus_consent_automation_id"), table_name="plus_consent_automation"
+    )
+    op.drop_table("plus_consent_automation")

src/fides/api/db/base.py

@@ -7,6 +7,7 @@
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail
 from fides.api.models.connectionconfig import ConnectionConfig
+from fides.api.models.consent_automation import ConsentAutomation
 from fides.api.models.custom_asset import CustomAsset
 from fides.api.models.custom_connector_template import CustomConnectorTemplate
 from fides.api.models.datasetconfig import DatasetConfig

src/fides/api/models/connectionconfig.py

@@ -7,14 +7,15 @@
 from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlalchemy.ext.mutable import MutableDict
-from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm import RelationshipProperty, Session, relationship
 from sqlalchemy_utils.types.encrypted.encrypted_type import (
     AesGcmEngine,
     StringEncryptedType,
 )
 
 from fides.api.common_exceptions import KeyOrNameAlreadyExists
 from fides.api.db.base_class import Base, FidesBase, JSONTypeOverride
+from fides.api.models.consent_automation import ConsentAutomation
 from fides.api.models.sql_models import System  # type: ignore[attr-defined]
 from fides.api.schemas.policy import ActionType
 from fides.api.schemas.saas.saas_config import SaaSConfig
@@ -164,6 +165,10 @@ class ConnectionConfig(Base):
 
     system = relationship(System, back_populates="connection_configs", uselist=False)
 
+    consent_automation: RelationshipProperty[Optional[ConsentAutomation]] = (
+        relationship(ConsentAutomation, cascade="all, delete-orphan")
+    )
+
     # Identifies the privacy actions needed from this connection by the associated system.
     enabled_actions = Column(
         ARRAY(Enum(ActionType, native_enum=False)), unique=False, nullable=True

src/fides/api/models/consent_automation.py

@@ -0,0 +1,168 @@
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+from sqlalchemy import Column, ForeignKey, String, UniqueConstraint
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import RelationshipProperty, Session, relationship
+
+from fides.api.db.base_class import Base, FidesBase  # type: ignore[attr-defined]
+from fides.api.models.privacy_notice import PrivacyNotice
+
+
+class ConsentAutomation(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "plus_consent_automation"
+
+    connection_config_id = Column(
+        String,
+        ForeignKey("connectionconfig.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+    )
+    consentable_items: "RelationshipProperty[List[ConsentableItem]]" = relationship(
+        "ConsentableItem",
+        back_populates="consent_automation",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+    )
+
+    @classmethod
+    def create(
+        cls, db: Session, *, data: Dict[str, Any], check_name: bool = False
+    ) -> "ConsentAutomation":
+        consentable_items = data.pop("consentable_items", [])
+        consent_automation = super().create(db=db, data=data, check_name=check_name)
+        link_consentable_items_to_consent_automation(
+            db, consentable_items, consent_automation
+        )
+        return consent_automation
+
+    def update(self, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation":
+        consentable_items = data.pop("consentable_items", [])
+        super().update(db=db, data=data)
+        link_consentable_items_to_consent_automation(db, consentable_items, self)
+        return self
+
+    @classmethod
+    def create_or_update(cls, db: Session, *, data: Dict[str, Any]) -> "ConsentAutomation":  # type: ignore[override]
+        consent_automation = ConsentAutomation.filter(
+            db=db,
+            conditions=(
+                ConsentAutomation.connection_config_id == data["connection_config_id"]
+            ),
+        ).first()
+
+        if consent_automation:
+            consent_automation.update(db=db, data=data)
+        else:
+            consent_automation = cls.create(db=db, data=data)
+
+        return consent_automation
+
+
+def link_consentable_items_to_consent_automation(
+    db: Session,
+    consentable_items: List[Dict[str, Any]],
+    consent_automation: ConsentAutomation,
+) -> None:
+    """
+    Takes a hierarchical list of consentable items and maps them to database items.
+    Attaches the database items to the consent automation.
+    """
+
+    existing_items = {
+        (item.type, str(item.external_id)): item
+        for item in consent_automation.consentable_items
+    }
+
+    def process_items(
+        items_data: List[Dict[str, Any]], parent_id: str = None
+    ) -> List[ConsentableItem]:
+        processed_items = []
+        for item_data in items_data:
+            external_id = str(item_data["id"])
+            item_type = item_data["type"]
+
+            key = (item_type, external_id)
+            item = existing_items.get(key)
+
+            if item:
+                item.notice_id = item_data.get("notice_id")
+            else:
+                item = ConsentableItem(
+                    external_id=external_id,
+                    consent_automation_id=consent_automation.id,
+                    parent_id=parent_id,
+                    type=item_type,
+                    name=item_data["name"],
+                    notice_id=item_data.get("notice_id"),
+                )
+                db.add(item)
+                # flush to the DB so we can get the auto-generated ID for this item
+                db.flush()
+
+            processed_items.append(item)
+
+            if "children" in item_data:
+                processed_items.extend(
+                    process_items(item_data["children"], parent_id=item.id)
+                )
+
+        return processed_items
+
+    try:
+        consent_automation.consentable_items = process_items(consentable_items)
+        db.commit()
+    except IntegrityError as exc:
+        logger.error("Error occurred while attempting to save consentable items", exc)
+
+    db.refresh(consent_automation)
+
+
+class ConsentableItem(Base):
+    @declared_attr
+    def __tablename__(self) -> str:
+        return "plus_consentable_item"
+
+    id = Column(
+        String(255), primary_key=True, index=True, default=FidesBase.generate_uuid
+    )
+
+    consent_automation_id = Column(
+        String,
+        ForeignKey(ConsentAutomation.id_field_path, ondelete="CASCADE"),
+        nullable=False,
+    )
+    external_id = Column(
+        String,
+        nullable=False,
+    )
+    parent_id = Column(
+        String,
+        ForeignKey("plus_consentable_item.id", ondelete="CASCADE"),
+        nullable=True,
+    )
+    notice_id = Column(String, ForeignKey(PrivacyNotice.id_field_path), nullable=True)
+    type = Column(
+        String,
+        nullable=False,
+    )
+    name = Column(String, nullable=False)
+    consent_automation = relationship(
+        "ConsentAutomation", back_populates="consentable_items"
+    )
+    children: "RelationshipProperty[List[ConsentableItem]]" = relationship(
+        "ConsentableItem",
+        back_populates="parent",
+        cascade="all, delete-orphan",
+        passive_deletes=True,
+    )
+    parent: "RelationshipProperty[Optional[ConsentableItem]]" = relationship(
+        "ConsentableItem",
+        back_populates="children",
+        remote_side=[id],
+    )
+
+    __table_args__ = (UniqueConstraint("consent_automation_id", "type", "external_id"),)

src/fides/api/schemas/consentable_item.py

@@ -0,0 +1,75 @@
+from typing import List, Optional, Type
+
+from pydantic import Field
+
+from fides.api.models.consent_automation import ConsentableItem as ConsentableItemModel
+from fides.api.schemas.base_class import FidesSchema
+
+
+class ConsentableItem(FidesSchema):
+    """
+    Schema to represent 3rd-party consentable items and privacy notice relationships.
+    """
+
+    id: str
+    type: str
+    name: str
+    notice_id: Optional[str] = None
+    children: List["ConsentableItem"] = Field(default_factory=list)
+    unmapped: Optional[bool] = False
+
+    @classmethod
+    def from_orm(
+        cls: Type["ConsentableItem"], obj: ConsentableItemModel
+    ) -> "ConsentableItem":
+        item = cls(
+            id=obj.external_id,
+            type=obj.type,
+            name=obj.name,
+            notice_id=obj.notice_id,
+        )
+        # recursively set children
+        item.children = [cls.from_orm(child) for child in getattr(obj, "children", [])]
+        return item
+
+
+def merge_consentable_items(
+    api_items: List[ConsentableItem], db_items: Optional[List[ConsentableItem]] = None
+) -> List[ConsentableItem]:
+    """
+    Recursively merges the lists of consentable items, setting the unmapped flag if the item is not in the database.
+
+    WARNING: This is a destructive operation for the api_items parameter.
+    """
+
+    if db_items is None:
+        db_items = []
+
+    def merge_consentable_items_recursive(
+        source: ConsentableItem, target: Optional[ConsentableItem]
+    ) -> None:
+        if target is None:
+            source.unmapped = True
+            for child in source.children:
+                child.unmapped = True
+            return
+
+        if source.id == target.id:
+            source.unmapped = False
+            source.notice_id = target.notice_id
+            target_children_map = {child.id: child for child in target.children}
+
+            for child in source.children:
+                target_child = target_children_map.get(child.id)
+                if target_child is not None:
+                    merge_consentable_items_recursive(child, target_child)
+
+    # create a map of target items for efficient lookup
+    db_item_map = {item.id: item for item in db_items}
+
+    # iterate through API items and merge
+    for api_item in api_items:
+        target_item = db_item_map.get(api_item.id)
+        merge_consentable_items_recursive(api_item, target_item)
+
+    return api_items