saas request overrides (#986)

* initial cut of saas request overrides. include mailchimp as an example and test case. minor refactor of some of the saas request execution to enable smoother override

* fix rebase issue by moving saas override tests into ops subdir

* import path updates to resolve conflicts caused by rebase

* add session parameter into graph task calls to fix saas override integration tests caused by rebase

* update changelog

* tweaks to saas connector overrides and associated tests

* expose override factory register as module variable for clenaer decorator calls

Co-authored-by: Adam Sachs <adam@Adams-MacBook-Pro.local>

CHANGELOG.md

@@ -29,6 +29,7 @@ The types of changes are:
 * Retry a DSR (FE) [#863](https://github.com/ethyca/fidesops/pull/938)
 * Add a Connection - Select a connector to configure (front end) [#760] (https://github.com/ethyca/fidesops/pull/987)
 * Add a Connection - Front End layout structure [#866] (https://github.com/ethyca/fidesops/pull/987)
+* Enable python function overrides for SaaS connector request execution [#986](https://github.com/ethyca/fidesops/pull/986)
 
 ### Changed
 

data/saas/config/request_override/mailchimp_override_config.yml

@@ -0,0 +1,81 @@
+saas_config:
+  fides_key: mailchimp_override_connector_example
+  name: Mailchimp SaaS Override Config
+  type: mailchimp
+  description: A sample schema representing the Mailchimp connector for Fidesops that includes request function overrides
+  version: 0.0.1
+
+  connector_params:
+    - name: domain
+    - name: username
+    - name: api_key
+
+  client_config:
+    protocol: https
+    host: <domain>
+    authentication:
+      strategy: basic
+      configuration:
+        username: <username>
+        password: <api_key>
+
+  test_request:
+    method: GET
+    path: /3.0/lists
+
+  endpoints:
+    - name: messages
+      requests:
+        read:
+          request_override: mailchimp_messages_access
+          param_values:
+            - name: conversation_id
+              references:
+                - dataset: mailchimp_override_connector_example
+                  field: conversations.id
+                  direction: from
+    - name: conversations
+      requests:
+        read:
+          method: GET
+          path: /3.0/conversations
+          query_params:
+            - name: count
+              value: 1000
+            - name: offset
+              value: 0
+          param_values:
+            - name: placeholder
+              identity: email
+          data_path: conversations
+          pagination:
+            strategy: offset
+            configuration:
+              incremental_param: offset
+              increment_by: 1000
+              limit: 10000
+    - name: member
+      requests:
+        read:
+          method: GET
+          path: /3.0/search-members
+          query_params:
+            - name: query
+              value: <email>
+          param_values:
+            - name: email
+              identity: email
+          data_path: exact_matches.members
+        update:
+          request_override: mailchimp_member_update
+          param_values:
+            - name: list_id
+              references:
+                - dataset: mailchimp_override_connector_example
+                  field: member.list_id
+                  direction: from
+            - name: subscriber_hash
+              references:
+                - dataset: mailchimp_override_connector_example
+                  field: member.id
+                  direction: from

data/saas/dataset/request_override/mailchimp_override_dataset.yml

@@ -0,0 +1,145 @@
+dataset:
+  - fides_key: mailchimp_override_connector_example
+    name: Mailchimp Override Dataset
+    description: A sample dataset representing the Mailchimp connector for Fidesops
+    collections:
+      - name: messages
+        fields:
+          - name: id
+            data_categories: [system.operations]
+          - name: conversation_id
+            data_categories: [system.operations]
+          - name: from_label
+            data_categories: [system.operations]
+          - name: from_email
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: subject
+            data_categories: [system.operations]
+          - name: message
+            data_categories: [user.provided.nonidentifiable]
+          - name: read
+            data_categories: [system.operations]
+          - name: timestamp
+            data_categories: [system.operations]
+      - name: conversations
+        fields:
+          - name: id
+            data_categories: [system.operations]
+          - name: campaign_id
+            data_categories: [system.operations]
+          - name: list_id
+            data_categories: [system.operations]
+          - name: from_email
+            data_categories: [user.provided.identifiable.contact.email]
+          - name: from_label
+            data_categories: [user.provided.identifiable.contact.email]
+          - name: subject
+            data_categories: [user.provided.nonidentifiable]
+      - name: member
+        fields:
+          - name: id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              primary_key: True
+          - name: list_id
+            data_categories: [system.operations]
+          - name: email_address
+            data_categories: [user.provided.identifiable.contact.email]
+            fidesops_meta:
+              data_type: string
+          - name: unique_email_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              data_type: string
+          - name: web_id
+            data_categories: [user.derived.identifiable.unique_id]
+            fidesops_meta:
+              data_type: string
+          - name: email_type
+            data_categories: [system.operations]
+          - name: status
+            data_categories: [system.operations]
+          - name: merge_fields
+            fields:
+              - name: FNAME
+                data_categories: [user.provided.identifiable.name]
+                fidesops_meta:
+                  data_type: string
+              - name: LNAME
+                data_categories: [user.provided.identifiable.name]
+                fidesops_meta:
+                  data_type: string
+              - name: ADDRESS
+                fields:
+                  - name: addr1
+                    data_categories: [user.provided.identifiable.contact.street]
+                    fidesops_meta:
+                      data_type: string
+                  - name: addr2
+                    data_categories: [user.provided.identifiable.contact.street]
+                    fidesops_meta:
+                      data_type: string
+                  - name: city
+                    data_categories: [user.provided.identifiable.contact.city]
+                    fidesops_meta:
+                      data_type: string
+                  - name: state
+                    data_categories: [user.provided.identifiable.contact.state]
+                    fidesops_meta:
+                      data_type: string
+                  - name: zip
+                    data_categories: [user.provided.identifiable.contact.postal_code]
+                    fidesops_meta:
+                      data_type: string
+                  - name: country
+                    data_categories: [user.provided.identifiable.contact.country]
+                    fidesops_meta:
+                      data_type: string
+              - name: PHONE
+                data_categories: [user.provided.identifiable.contact.phone_number]
+                fidesops_meta:
+                  data_type: string
+              - name: BIRTHDAY
+                data_categories: [user.provided.identifiable.date_of_birth]
+                fidesops_meta:
+                  data_type: string
+          - name: ip_signup
+            data_categories: [user.derived.identifiable.device.ip_address]
+            fidesops_meta:
+              data_type: string
+          - name: timestamp_signup
+            data_categories: [system.operations]
+          - name: ip_opt
+            data_categories: [system.operations]
+          - name: timestamp_opt
+            data_categories: [system.operations]
+          - name: language
+            data_categories: [system.operations]
+          - name: email_client
+            data_categories: [system.operations]
+          - name: location
+            fields:
+              - name: latitude
+                data_categories: [system.operations]
+              - name: longitude
+                data_categories: [system.operations]
+              - name: gmtoff
+                data_categories: [system.operations]
+              - name: dstoff
+                data_categories: [system.operations]
+              - name: country_code
+                data_categories: [user.provided.identifiable.contact.country]
+                fidesops_meta:
+                  data_type: string
+              - name: timezone
+                data_categories: [system.operations]
+          - name: source
+            data_categories: [system.operations]
+          - name: tags
+            fields:
+              - name: id
+                data_categories: [system.operations]
+              - name: name
+                data_categories: [system.operations]

pytest.ini

@@ -14,6 +14,7 @@ markers =
     integration_mariadb
     integration_bigquery
     integration_saas
+    integration_saas_override
     integration_mailchimp
     integration_zendesk
     integration_sentry

src/fidesops/common_exceptions.py

@@ -143,3 +143,11 @@ class NoSuchStrategyException(ValueError):
 
 class FunctionalityNotConfigured(Exception):
     """Custom exception for when invoked functionality is unavailable due to configuration."""
+
+
+class InvalidSaaSRequestOverrideException(ValueError):
+    """Exception for when a provied SaaS request override function is invalid"""
+
+
+class NoSuchSaaSRequestOverrideException(ValueError):
+    """Exception for when a requested SaaS request override function does not exist"""

src/fidesops/schemas/saas/saas_config.py

@@ -86,8 +86,9 @@ class SaaSRequest(BaseModel):
     Includes optional strategies for postprocessing and pagination.
     """
 
-    path: str
-    method: HTTPMethod
+    request_override: Optional[str]
+    path: Optional[str]
+    method: Optional[HTTPMethod]
     headers: Optional[List[Header]] = []
     query_params: Optional[List[QueryParam]] = []
     body: Optional[str]
@@ -159,6 +160,34 @@ def validate_grouped_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]:
 
         return values
 
+    @root_validator
+    def validate_override(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate that configs related to request overrides are set properly"""
+        if not values.get("request_override"):
+            if not values.get("path"):
+                raise ValueError(
+                    "A request must specify a path if no request_override is provided"
+                )
+            if not values.get("method"):
+                raise ValueError(
+                    "A request must specify a method if no request_override is provided"
+                )
+
+        else:  # if a request override is specified, many fields are NOT allowed
+            invalid = [
+                k
+                for k in values.keys()
+                if values.get(k)
+                and k not in ("request_override", "param_values", "grouped_inputs")
+            ]
+            if invalid:
+                invalid_joined = ", ".join(invalid)
+                raise ValueError(
+                    f"Invalid properties [{invalid_joined}] on a request with request_override specified"
+                )
+
+        return values
+
 
 class Endpoint(BaseModel):
     """A collection of read/update/delete requests which corresponds to a FidesopsDataset collection (by name)"""