Merge branch 'main' into jpople/prod-2233/monitor-results-pagination-fix

.github/workflows/backend_checks.yml

@@ -79,8 +79,8 @@ jobs:
       matrix:
         session_name:
           [
-            "isort",
-            "black",
+            '"isort(check)"',
+            '"black(check)"',
             "mypy",
             "pylint",
             "xenon",

CHANGELOG.md

@@ -26,6 +26,7 @@ The types of changes are:
 - Invite users via email flow [#4539](https://github.com/ethyca/fides/pull/4539)
 - Added new Google Cloud SQL for Postgres Connector [#5014](https://github.com/ethyca/fides/pull/5014)
 - Added access and erasure support for the Twilio SMS integration [#4979](https://github.com/ethyca/fides/pull/4979)
+- Added erasure support for Snap integration [#5011](https://github.com/ethyca/fides/pull/5011)
 
 ### Changed
 - Navigation changes. 'Management' was renamed 'Settings'. Properties was moved to Settings section. [#5005](https://github.com/ethyca/fides/pull/5005)

data/saas/config/snap_config.yml

@@ -0,0 +1,92 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: Snap
+  type: snap
+  description: A sample schema representing the Snap integration for Fides
+  version: 0.1.0
+
+  connector_params:
+    - name: domain
+      label: Domain
+      default_value: adsapi.snapchat.com
+    - name: client_id
+      label: Snap client ID
+      sensitive: true
+    - name: client_secret
+      label: Snap client secret key
+      sensitive: true
+    - name: ad_account_id
+      label: Ad account ID
+    - name: redirect_uri
+      label: Redirect URL
+      description: The Fides URL to which users will be redirected upon successful authentication
+
+  client_config:
+    protocol: https
+    host: <domain>
+    authentication:
+      strategy: oauth2_authorization_code
+      configuration:
+        authorization_request:
+          method: GET
+          client_config:
+            protocol: https
+            host: accounts.snapchat.com
+          path: /login/oauth2/authorize
+          query_params:
+            - name: client_id
+              value: <client_id>
+            - name: state
+              value: <state>
+            - name: redirect_uri
+              value: <redirect_uri>
+            - name: response_type
+              value: code
+            - name: scope
+              value: snapchat-marketing-api
+        token_request:
+          method: POST
+          path: /login/oauth2/access_token
+          headers:
+            - name: Content-Type
+              value: application/x-www-form-urlencoded
+          body: |
+            {
+              "client_id": "<client_id>",
+              "client_secret": "<client_secret>",
+              "grant_type": "authorization_code",
+              "code": "<code>",
+              "scope": "snapchat-marketing-api"
+              "redirect_uri": "<redirect_uri>"
+            }
+        refresh_request:
+          method: POST
+          path: /login/oauth2/access_token
+          headers:
+            - name: Content-Type
+              value: application/x-www-form-urlencoded
+          body: |
+            {
+              "client_id": "<client_id>",
+              "client_secret": "<client_secret>",
+              "grant_type": "refresh_token",
+              "refresh_token": "<refresh_token>"
+            }
+
+  rate_limit_config:
+    limits:
+      - rate: 10
+        period: second
+
+  test_request:
+    method: GET
+    path: /v1/me
+
+  endpoints:
+    - name: user
+      requests:
+        delete:
+          request_override: snap_user_delete
+          param_values:
+            - name: email
+              identity: email

data/saas/dataset/snap_dataset.yml

@@ -0,0 +1,7 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: snap_copy
+    description: A sample dataset representing the Snap copy connector for Fides
+    collections:
+      - name: user
+        fields: []

noxfiles/ci_nox.py

@@ -24,8 +24,8 @@
 @nox.session()
 def static_checks(session: nox.Session) -> None:
     """Run the static checks only."""
-    session.notify("black")
-    session.notify("isort")
+    session.notify("black(fix)")
+    session.notify("isort(fix)")
     session.notify("xenon")
     session.notify("mypy")
     session.notify("pylint")
@@ -35,8 +35,8 @@ def static_checks(session: nox.Session) -> None:
 @nox.parametrize(
     "mode",
     [
-        nox.param("fix", id="fix"),
         nox.param("check", id="check"),
+        nox.param("fix", id="fix"),
     ],
 )
 def black(session: nox.Session, mode: str) -> None:
@@ -52,8 +52,8 @@ def black(session: nox.Session, mode: str) -> None:
 @nox.parametrize(
     "mode",
     [
-        nox.param("fix", id="fix"),
         nox.param("check", id="check"),
+        nox.param("fix", id="fix"),
     ],
 )
 def isort(session: nox.Session, mode: str) -> None:

noxfiles/setup_tests_nox.py

@@ -1,5 +1,7 @@
 from typing import Optional
 
+from nox import Session
+
 from constants_nox import (
     CI_ARGS_EXEC,
     COMPOSE_FILE,
@@ -11,7 +13,6 @@
     START_APP,
     START_APP_WITH_EXTERNAL_POSTGRES,
 )
-from nox import Session
 from run_infrastructure import API_TEST_DIR, OPS_TEST_DIR, run_infrastructure
 
 

pyproject.toml

@@ -103,6 +103,7 @@ exclude = '''
 ###########
 [tool.isort]
 known_first_party = ["versioneer"]
+known_third_party = ["nox"]
 profile = "black"
 line_length = 88
 src_paths = ["src", "tests", "noxfiles"]

src/fides/api/service/saas_request/override_implementations/snap_request_overrides.py

@@ -0,0 +1,85 @@
+import hashlib
+import json
+from typing import Any, Dict, List
+
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.service.saas_request.saas_request_override_factory import (
+    SaaSRequestType,
+    register,
+)
+
+
+def signed_payload(given_email: str) -> str:
+    """This function is to take in the value of our identity email, and then ensure it is all lower case, and then hash it with SHA256"""
+    sub_email = given_email.lower()
+    hash_value = hashlib.sha256(sub_email.encode())
+    sig = hash_value.hexdigest()
+
+    return sig
+
+
+@register("snap_user_delete", [SaaSRequestType.DELETE])
+def snap_user_delete(
+    client: AuthenticatedClient,
+    param_values_per_row: List[Dict[str, Any]],
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    secrets: Dict[str, Any],
+) -> int:
+    rows_deleted = 0
+    params = {"limit": 500}
+    get_organizations = client.send(
+        SaaSRequestParams(
+            method=HTTPMethod.GET,
+            path="/v1/me/organizations?with_ad_accounts=true",
+            params=params,
+        )
+    )
+    org_out = get_organizations.json()
+    ad_account_ids: List[str] = []
+    organizations = org_out.get("organizations", [])
+    for org_info in organizations:
+        organization = org_info.get("organization", {})
+        ad_accounts = organization.get("ad_accounts", [])
+        for ad_account in ad_accounts:
+            ad_account_ids.append(ad_account["id"])
+    for ad_account in ad_account_ids:
+        get_segments = client.send(
+            SaaSRequestParams(
+                method=HTTPMethod.GET,
+                path=f"/v1/adaccounts/{ad_account}/segments",
+                params=params,
+            )
+        )
+        ad_out = get_segments.json()
+        segments = ad_out.get("segments", [])
+        for segment_info in segments:
+            segment = segment_info.get("segment", {})
+            segment_id = segment.get("id")
+            for row_param_values in param_values_per_row:
+                email = row_param_values["email"]
+                payload = json.dumps(
+                    {
+                        "users": [
+                            {
+                                "schema": ["EMAIL_SHA256"],
+                                "data": [[signed_payload(email)]],
+                            }
+                        ]
+                    }
+                )
+                client.send(
+                    SaaSRequestParams(
+                        method=HTTPMethod.DELETE,
+                        path=f"/v1/segments/{segment_id}/users",
+                        headers={
+                            "Content-Type": "application/json",
+                        },
+                        body=payload,
+                    )
+                )
+                rows_deleted += 1
+    return rows_deleted

tests/fixtures/saas/snap_fixtures.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+
+from tests.ops.integration_tests.saas.connector_runner import (
+    ConnectorRunner,
+    generate_random_email,
+)
+from tests.ops.test_helpers.vault_client import get_secrets
+
+secrets = get_secrets("snap")
+
+
+@pytest.fixture(scope="session")
+def snap_secrets(saas_config) -> Dict[str, Any]:
+    return {
+        "domain": pydash.get(saas_config, "snap.domain") or secrets["domain"],
+        "client_id": pydash.get(saas_config, "snap.client_id")
+        or secrets["snap_client_id"],
+        "client_secret": pydash.get(saas_config, "snap.client_secret")
+        or secrets["snap_client_secret"],
+        "ad_account_id": pydash.get(saas_config, "snap.ad_account_id")
+        or secrets["ad_account_id"],
+        "redirect_uri": pydash.get(saas_config, "snap.redirect_uri")
+        or secrets["redirect_uri"],
+        "access_token": pydash.get(saas_config, "snap.access_token")
+        or secrets["access_token"],
+    }
+
+
+@pytest.fixture(scope="session")
+def snap_identity_email(saas_config) -> str:
+    return pydash.get(saas_config, "snap.identity_email") or secrets["identity_email"]
+
+
+@pytest.fixture
+def snap_erasure_identity_email() -> str:
+    return generate_random_email()
+
+
+@pytest.fixture
+def snap_runner(db, cache, snap_secrets) -> ConnectorRunner:
+    return ConnectorRunner(
+        db,
+        cache,
+        "snap",
+        snap_secrets,
+    )