252 saas connector sendgrid (#883)

* initial sendgrid saas connector integration. access only, contacts only

* erasure (update) support for sendgrid

* remove unused imports

* update sendgrid test fixture to expect 404 response status code because of ignore_errors enhancement

* Fixing import order and cleaning up the retry logic for consistency

* add sendgrid env var support to makefile and unsafe_pr_checks config. remove DELETE endpoint per PR comments

* Added delete endpoint for contacts

* Fixing data_path for contacts endpoint

* Reverting search query to improve performance and avoid server timeouts

* Updated delete endpoint request, used request instead of SaaSRequest in tests

* updated imports after check suggestion

* Updated code after review

* Removed unused variables, imports

* Restoring Makefile

* Fixed import cryptographic_util error

* Misc fixes

* Updated Changelog file

* Updated Changelog for unreleased section and pulled main

* Updated Changelog and added Sendgrid in unreleased section

* Updated Changelog and added Sendgrid in unreleased section with link

* Updated Changelog and added Sendgrid in added section after Adam's suggestion

Co-authored-by: Adam Sachs <adam@Adams-MacBook-Pro.local>
Co-authored-by: Adam Sachs <adam@Adams-MBP.attlocal.net>
Co-authored-by: Adrian Galvan <adriang430@gmail.com>
Co-authored-by: Hamza W <hamza@Hamzas-MacBook-Pro.local>
Co-authored-by: Adrian Galvan <adrian@ethyca.com>

.github/workflows/unsafe_pr_checks.yml

@@ -23,7 +23,6 @@ jobs:
           SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }}
         run: make pytest-integration-external
 
-
   External-SaaS-Connectors:
     runs-on: ubuntu-latest
     if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks')

CHANGELOG.md

@@ -20,6 +20,7 @@ The types of changes are:
 
 ### Added
 * Erasure support for Salesforce [#888](https://github.com/ethyca/fidesops/pull/888)
+* Access and erasure support for Sendgrid contacts endpoint [#883](https://github.com/ethyca/fidesops/pull/883)
 
 ### Breaking Changes
 

data/saas/config/sendgrid_config.yml

@@ -0,0 +1,46 @@
+saas_config:
+  fides_key: sendgrid_connector_example
+  name: Sendgrid SaaS Config
+  type: sendgrid
+  description: A sample schema representing the Sendgrid connector for Fidesops
+  version: 0.0.1
+
+  connector_params:
+    - name: domain
+    - name: api_key
+
+  client_config:
+    protocol: https
+    host: <domain>
+    authentication:
+      strategy: bearer
+      configuration:
+        token: <api_key>
+
+  test_request:
+    method: GET
+    path: /v3/marketing/contacts
+
+  endpoints:
+    - name: contacts
+      requests:
+        read:
+          method: POST
+          path: /v3/marketing/contacts/search
+          body: |
+            {
+              "query": "email = '<email>'"
+            }
+          param_values:
+            - name: email
+              identity: email
+          data_path: result
+        delete:
+          method: DELETE
+          path: /v3/marketing/contacts?ids=<contact_id>
+          param_values:
+            - name: contact_id
+              references:
+                - dataset: sendgrid_connector_example
+                  field: contacts.id
+                  direction: from

data/saas/dataset/sendgrid_dataset.yml

@@ -0,0 +1,88 @@
+dataset:
+  - fides_key: sendgrid_connector_example
+    name: Sendgrid Dataset
+    description: A sample dataset representing the Sendgrid connector for Fidesops
+    collections:
+      - name: contacts
+        fields:
+          - name: id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              data_type: string
+              primary_key: True
+          - name: first_name
+            data_categories: [user.provided.identifiable.name]
+            fidesops_meta:
+              data_type: string
+          - name: last_name
+            data_categories: [user.provided.identifiable.name]
+            fidesops_meta:
+              data_type: string
+          - name: email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: alternate_emails
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              data_type: string[]
+          - name: address_line_1
+            data_categories: [user.provided.identifiable.contact.street]
+            fidesops_meta:
+              data_type: string
+          - name: address_line_2
+            data_categories: [user.provided.identifiable.contact.street]
+            fidesops_meta:
+              data_type: string
+          - name: city
+            data_categories: [user.provided.identifiable.contact.city]
+            fidesops_meta:
+              data_type: string
+          - name: state_province_region
+            data_categories: [user.provided.identifiable.contact.state]
+            fidesops_meta:
+              data_type: string
+          - name: country
+            data_categories: [user.provided.identifiable.contact.country]
+            fidesops_meta:
+              data_type: string
+          - name: postal_code
+            data_categories: [user.provided.identifiable.contact.postal_code]
+            fidesops_meta:
+              data_type: string
+          - name: phone_number
+            data_categories: [user.provided.identifiable.contact.phone_number]
+            fidesops_meta:
+              data_type: string
+          - name: whatsapp
+            data_categories: [user.provided.identifiable.contact.phone_number]
+            fidesops_meta:
+              data_type: string
+          - name: list_ids
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string[]
+          - name: segment_ids
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string[]
+          - name: created_at
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: updated_at
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+      # - name: lists
+      #   fields:
+      #   - name: id
+      #     data_categories: [system.operations]
+      #     fidesops_meta:
+      #       primary_key: True
+      #   - name: name
+      #     data_categories: [user.provided.nonidentifiable] # not sure about this?
+      #     fidesops_meta:
+      #       data_type: string
+
+

pytest.ini

@@ -20,6 +20,7 @@ markers =
     integration_stripe
     integration_hubspot
     integration_segment
+    integration_sendgrid
     integration_outreach
     integration_salesforce
     unit_saas

saas_config.toml

@@ -43,6 +43,11 @@ access_secret = ""
 user_token = ""
 identity_email = ""
 
+[sendgrid]
+domain = ""
+api_key = ""
+identity_email = ""
+
 [outreach]
 domain = ""
 client_id = ""
@@ -65,4 +70,4 @@ identity_email = ""
 domain = ""
 username = ""
 api_key = ""
-identity_email = ""
+identity_email = ""

src/fidesops/schemas/saas/saas_config.py

@@ -190,6 +190,7 @@ class SaaSType(Enum):
     stripe = "stripe"
     zendesk = "zendesk"
     custom = "custom"
+    sendgrid = "sendgrid"
 
 
 class SaaSConfig(BaseModel):

tests/conftest.py

@@ -42,6 +42,7 @@
 from .fixtures.saas.outreach_fixtures import *
 from .fixtures.saas.salesforce_fixtures import *
 from .fixtures.saas.segment_fixtures import *
+from .fixtures.saas.sendgrid_fixtures import *
 from .fixtures.saas.sentry_fixtures import *
 from .fixtures.saas.stripe_fixtures import *
 from .fixtures.saas.zendesk_fixtures import *

tests/fixtures/saas/sendgrid_fixtures.py

@@ -0,0 +1,163 @@
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+import requests
+from fideslib.core.config import load_toml
+from fideslib.cryptography import cryptographic_util
+from fideslib.db import session
+from sqlalchemy.orm import Session
+from starlette.status import HTTP_202_ACCEPTED
+
+from fidesops.models.connectionconfig import (
+    AccessLevel,
+    ConnectionConfig,
+    ConnectionType,
+)
+from fidesops.models.datasetconfig import DatasetConfig
+from tests.fixtures.application_fixtures import load_dataset
+from tests.fixtures.saas_example_fixtures import load_config
+from tests.test_helpers.saas_test_utils import poll_for_existence
+from tests.test_helpers.vault_client import get_secrets
+
+saas_config = load_toml(["saas_config.toml"])
+secrets = get_secrets("sendgrid")
+
+SENDGRID_ERASURE_FIRSTNAME = "Erasurefirstname"
+
+
+@pytest.fixture(scope="session")
+def sendgrid_erasure_identity_email():
+    return f"{cryptographic_util.generate_secure_random_string(13)}@email.com"
+
+
+@pytest.fixture(scope="function")
+def sendgrid_secrets():
+    return {
+        "domain": pydash.get(saas_config, "sendgrid.domain") or secrets["domain"],
+        "api_key": pydash.get(saas_config, "sendgrid.api_key") or secrets["api_key"],
+    }
+
+
+@pytest.fixture(scope="function")
+def sendgrid_identity_email():
+    return (
+        pydash.get(saas_config, "sendgrid.identity_email") or secrets["identity_email"]
+    )
+
+
+@pytest.fixture
+def sendgrid_config() -> Dict[str, Any]:
+    return load_config("data/saas/config/sendgrid_config.yml")
+
+
+@pytest.fixture
+def sendgrid_dataset() -> Dict[str, Any]:
+    return load_dataset("data/saas/dataset/sendgrid_dataset.yml")[0]
+
+
+@pytest.fixture(scope="function")
+def sendgrid_connection_config(
+    db: session, sendgrid_config, sendgrid_secrets
+) -> Generator:
+    fides_key = sendgrid_config["fides_key"]
+    connection_config = ConnectionConfig.create(
+        db=db,
+        data={
+            "key": fides_key,
+            "name": fides_key,
+            "connection_type": ConnectionType.saas,
+            "access": AccessLevel.write,
+            "secrets": sendgrid_secrets,
+            "saas_config": sendgrid_config,
+        },
+    )
+    yield connection_config
+    connection_config.delete(db)
+
+
+@pytest.fixture
+def sendgrid_dataset_config(
+    db: Session,
+    sendgrid_connection_config: ConnectionConfig,
+    sendgrid_dataset: Dict[str, Any],
+) -> Generator:
+    fides_key = sendgrid_dataset["fides_key"]
+    sendgrid_connection_config.name = fides_key
+    sendgrid_connection_config.key = fides_key
+    sendgrid_connection_config.save(db=db)
+    dataset = DatasetConfig.create(
+        db=db,
+        data={
+            "connection_config_id": sendgrid_connection_config.id,
+            "fides_key": fides_key,
+            "dataset": sendgrid_dataset,
+        },
+    )
+    yield dataset
+    dataset.delete(db=db)
+
+
+@pytest.fixture(scope="function")
+def sendgrid_erasure_data(
+    sendgrid_connection_config, sendgrid_erasure_identity_email, sendgrid_secrets
+) -> Generator:
+    """
+    Creates a dynamic test data record for erasure tests.
+    Yields contact ID as this may be useful to have in test scenarios
+    """
+
+    base_url = f"https://{sendgrid_secrets['domain']}"
+    # Create contact
+    body = {
+        "list_ids": ["62d20902-1cdd-42e7-8d5d-0fbb2a8be13e"],
+        "contacts": [
+            {
+                "address_line_1": "address_line_1",
+                "address_line_2": "address_line_2",
+                "city": "CITY (optional)",
+                "country": "country (optional)",
+                "email": sendgrid_erasure_identity_email,
+                "first_name": SENDGRID_ERASURE_FIRSTNAME,
+                "last_name": "Testcontact",
+                "postal_code": "postal_code (optional)",
+                "state_province_region": "state (optional)",
+                "custom_fields": {},
+            }
+        ],
+    }
+    headers = {"Authorization": f"Bearer {sendgrid_secrets['api_key']}"}
+    contacts_response = requests.put(
+        url=f"{base_url}/v3/marketing/contacts", json=body, headers=headers
+    )
+    assert HTTP_202_ACCEPTED == contacts_response.status_code
+    error_message = f"Contact with email {sendgrid_erasure_identity_email} could not be added to Sendgrid"
+    contact = poll_for_existence(
+        contact_exists,
+        (sendgrid_erasure_identity_email, sendgrid_secrets),
+        error_message=error_message,
+    )
+    yield contact
+
+
+def contact_exists(sendgrid_erasure_identity_email: str, sendgrid_secrets):
+    """
+    Confirm whether contact exists by calling contact search by email api and comparing resulting firstname str.
+    Returns contact ID if it exists, returns None if it does not.
+    """
+    base_url = f"https://{sendgrid_secrets['domain']}"
+    body = {"emails": [sendgrid_erasure_identity_email]}
+    headers = {
+        "Authorization": f"Bearer {sendgrid_secrets['api_key']}",
+    }
+
+    contact_response = requests.post(
+        url=f"{base_url}/v3/marketing/contacts/search/emails",
+        headers=headers,
+        json=body,
+    )
+    # we expect 404 if contact doesn't exist
+    if 404 == contact_response.status_code:
+        return None
+
+    return contact_response.json()["result"][sendgrid_erasure_identity_email]["contact"]