[SaaS Connector] Salesforce (erasure) (#888)

CHANGELOG.md

@@ -18,6 +18,9 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.3...main)
 
+### Added
+* Erasure support for Salesforce [#888](https://github.com/ethyca/fidesops/pull/888)
+
 ### Breaking Changes
 
 * Update fidesops to use bcrypt for hashing [#876](https://github.com/ethyca/fidesops/pull/876)

data/saas/config/salesforce_config.yml

@@ -49,6 +49,19 @@ saas_config:
                 - dataset: salesforce_connector_example
                   field: contact_list.Id
                   direction: from
+        update:
+          method: PATCH
+          path: /services/data/v54.0/sobjects/Contact/<contact_id>
+          body: |
+            {
+              <masked_object_fields>
+            }
+          param_values:
+            - name: contact_id
+              references:
+                - dataset: salesforce_connector_example
+                  field: contacts.Id
+                  direction: from
     - name: case_list
       requests:
         read:
@@ -75,6 +88,19 @@ saas_config:
                 - dataset: salesforce_connector_example
                   field: case_list.Id
                   direction: from
+        update:
+          method: PATCH
+          path: /services/data/v54.0/sobjects/Case/<case_id>
+          body: |
+            {
+              <masked_object_fields>
+            }
+          param_values:
+            - name: case_id
+              references:
+                - dataset: salesforce_connector_example
+                  field: cases.Id
+                  direction: from
     - name: lead_list
       requests:
         read:
@@ -98,6 +124,19 @@ saas_config:
                 - dataset: salesforce_connector_example
                   field: lead_list.Id
                   direction: from
+        update:
+          method: PATCH
+          path: /services/data/v54.0/sobjects/Lead/<lead_id>
+          body: |
+            {
+              <masked_object_fields>
+            }
+          param_values:
+            - name: lead_id
+              references:
+                - dataset: salesforce_connector_example
+                  field: leads.Id
+                  direction: from
     - name: accounts
       requests:
         read:
@@ -108,6 +147,19 @@ saas_config:
               references:
                 - dataset: salesforce_connector_example
                   field: contacts.AccountId
+        update:
+          method: PATCH
+          path: /services/data/v54.0/sobjects/Account/<account_id>
+          body: |
+            {
+              <masked_object_fields>
+            }
+          param_values:
+            - name: account_id
+              references:
+                - dataset: salesforce_connector_example
+                  field: accounts.Id
+                  direction: from
     - name: campaign_member_list
       requests:
         read:
@@ -131,3 +183,16 @@ saas_config:
                 - dataset: salesforce_connector_example
                   field: campaign_member_list.Id
                   direction: from
+        update:
+          method: PATCH
+          path: /services/data/v54.0/sobjects/CampaignMember/<campaign_member_id>
+          body: |
+            {
+              <masked_object_fields>
+            }
+          param_values:
+            - name: campaign_member_id
+              references:
+                - dataset: salesforce_connector_example
+                  field: campaign_members.Id
+                  direction: from

data/saas/dataset/salesforce_dataset.yml

@@ -35,6 +35,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: True
           - name: IsDeleted
             data_categories: [system.operations]
             fidesops_meta:
@@ -63,6 +64,7 @@ dataset:
             data_categories: [user.provided.identifiable.name]
             fidesops_meta:
               data_type: string
+              read_only: True
           - name: OtherStreet
             data_categories: [user.provided.identifiable.contact.street]
             fidesops_meta:
@@ -281,6 +283,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: True
           - name: IsDeleted
             data_categories: [system.operations]
             fidesops_meta:
@@ -443,6 +446,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: True
           - name: IsDeleted
             data_categories: [system.operations]
             fidesops_meta:
@@ -467,6 +471,7 @@ dataset:
             data_categories: [user.provided.identifiable.name]
             fidesops_meta:
               data_type: string
+              read_only: True
           - name: Title
             data_categories: [system.operations]
             fidesops_meta:
@@ -673,6 +678,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: True
           - name: IsDeleted
             data_categories: [system.operations]
             fidesops_meta:
@@ -723,14 +729,17 @@ dataset:
             data_categories: [user.provided.identifiable.name]
             fidesops_meta:
               data_type: string
+              read_only: True
           - name: FirstName
             data_categories: [user.provided.identifiable.name]
             fidesops_meta:
               data_type: string
+              read_only: True
           - name: LastName
             data_categories: [user.provided.identifiable.name]
             fidesops_meta:
               data_type: string
+              read_only: True
           - name: Title
             data_categories: [system.operations]
             fidesops_meta:
@@ -823,6 +832,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: True
           - name: IsDeleted
             data_categories: [system.operations]
             fidesops_meta:

src/fidesops/service/connectors/saas_query_config.py

@@ -161,7 +161,7 @@ def generate_query(
 
         return saas_request_params
 
-    def generate_update_stmt(
+    def generate_update_stmt(  # pylint: disable=R0914
         self, row: Row, policy: Policy, request: PrivacyRequest
     ) -> SaaSRequestParams:
         """
@@ -192,6 +192,11 @@ def generate_update_stmt(
                     self.secrets, param_value.connector_param
                 )
 
+        # remove any row values for fields marked as read-only, these will be omitted from all update maps
+        for field_path, field in self.field_map().items():
+            if field.read_only:
+                pydash.unset(row, field_path.string_path)
+
         # mask row values
         update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request)
         masked_object: Dict[str, Any] = unflatten_dict(update_value_map)
@@ -219,19 +224,20 @@ def generate_update_stmt(
 
     def all_value_map(self, row: Row) -> Dict[str, Any]:
         """
-        Takes a row and preserves only the fields that are defined in the Dataset
-        and are not flagged as read-only. Used for scenarios when an update endpoint
-        has required fields other than just the fields being updated.
+        Takes a row and preserves only the fields that are defined in the Dataset.
+        Used for scenarios when an update endpoint has required fields other than
+        just the fields being updated.
         """
         all_value_map: Dict[str, Any] = {}
         for field_path, field in self.field_map().items():
-            # only map scalar fields that are not read-only
-            if isinstance(field, ScalarField) and not field.read_only:
-                # only map if the value exists on the row
-                if pydash.get(row, field_path.string_path) is not None:
-                    all_value_map[field_path.string_path] = pydash.get(
-                        row, field_path.string_path
-                    )
+            # only map scalar fields
+            if (
+                isinstance(field, ScalarField)
+                and pydash.get(row, field_path.string_path) is not None
+            ):
+                all_value_map[field_path.string_path] = pydash.get(
+                    row, field_path.string_path
+                )
         return all_value_map
 
     def query_to_str(self, t: T, input_data: Dict[str, List[Any]]) -> str:

src/fidesops/util/saas_util.py

@@ -193,7 +193,7 @@ def assign_placeholders(value: Any, param_values: Dict[str, Any]) -> Optional[An
         placeholders = re.findall("<([^<>]+)>", value)
         for placeholder in placeholders:
             placeholder_value = param_values.get(placeholder)
-            if placeholder_value:
+            if placeholder_value is not None:
                 value = value.replace(f"<{placeholder}>", str(placeholder_value))
             else:
                 return None

tests/fixtures/saas/salesforce_fixtures.py

@@ -7,6 +7,7 @@
 from fideslib.cryptography import cryptographic_util
 from fideslib.db import session
 from sqlalchemy.orm import Session
+from starlette.status import HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND
 
 from fidesops.models.connectionconfig import (
     AccessLevel,
@@ -125,7 +126,7 @@ def salesforce_dataset_config(
 
 @pytest.fixture(scope="function")
 def salesforce_create_erasure_data(
-    salesforce_identity_email, salesforce_secrets
+    salesforce_erasure_identity_email, salesforce_secrets
 ) -> Generator:
     """
     Creates a dynamic test data record for tests.
@@ -151,7 +152,7 @@ def salesforce_create_erasure_data(
     contact_data = {
         "firstName": "Fidesops",
         "lastName": "Test Contact",
-        "email": salesforce_identity_email,
+        "email": salesforce_erasure_identity_email,
         "AccountId": account_id,
     }
     contacts_response = requests.post(
@@ -166,20 +167,21 @@ def salesforce_create_erasure_data(
     lead_data = {
         "firstName": "Fidesops",
         "lastName": "Test Lead",
-        "email": salesforce_identity_email,
+        "email": salesforce_erasure_identity_email,
         "Company": "Test Company",
     }
     leads_response = requests.post(
         url=f"{base_url}/services/data/v54.0/sobjects/Lead",
         headers=headers,
         json=lead_data,
     )
+
     assert leads_response.ok
     lead_id = leads_response.json()["id"]
 
     # Create Case
     case_data = {
-        "SuppliedEmail": salesforce_identity_email,
+        "SuppliedEmail": salesforce_erasure_identity_email,
         "SuppliedCompany": "Test Company",
         "ContactId": contact_id,
     }
@@ -218,6 +220,30 @@ def salesforce_create_erasure_data(
         json=campaign_member_data,
     )
     assert campaign_members_response.ok
+
     campaign_member_id = campaign_members_response.json()["id"]
 
-    yield contact_id, lead_id, case_id, account_id, campaign_member_id
+    yield account_id, contact_id, case_id, lead_id, campaign_member_id
+
+    # cleanup data by doing a full deletion instead of just masking
+    case_response = requests.delete(
+        url=f"{base_url}/services/data/v54.0/sobjects/Case/{case_id}", headers=headers
+    )
+    assert case_response.status_code == HTTP_204_NO_CONTENT
+
+    case_response = requests.get(
+        url=f"{base_url}/services/data/v54.0/sobjects/Case/{case_id}", headers=headers
+    )
+    assert case_response.status_code == HTTP_404_NOT_FOUND
+
+    account_response = requests.delete(
+        url=f"{base_url}/services/data/v54.0/sobjects/Account/{account_id}",
+        headers=headers,
+    )
+    assert account_response.status_code == HTTP_204_NO_CONTENT
+
+    account_response = requests.get(
+        url=f"{base_url}/services/data/v54.0/sobjects/Account/{account_id}",
+        headers=headers,
+    )
+    assert account_response.status_code == HTTP_404_NOT_FOUND