Endpoint: Return Secrets for a Connector Type [#753] (#795)

* Adds Saas type to saas yaml config

* alter postman collection

* updates changelog

* lint fixes

* Add endpoint to surface all available connectors including database options and saas options.

* Exclude custom and manual types from list of available connectors.

- Add docs and postman collection.

* Update changelog.

* Add an endpoint to fetch the types of secrets that should be supplied for a given connection type.

- Relocate "load_config" which we use to load saas config yamls, now that we have another use case beyond unit tests.

* Dynamically override the SaaSSchema docstring for a given saas connector type, so the description isn't abstract.

- Update changelog
- Add docs
- Add endpoint to postman collection

* Add missing import.

* Add a request method to docs.

* Update docstring.

* Remove committed ANALYTICS_ID.

* Import ClientDetail from fideslib instead of fidesops.

* Fix import order.

* Restore removed items in changelog.

Co-authored-by: eastandwestwind <eastandwestwind@gmail.com>

CHANGELOG.md

@@ -24,6 +24,7 @@ The types of changes are:
 * Erasure support for Zendesk [#775](https://github.com/ethyca/fidesops/pull/775)
 * Adds SaaS connection type to SaaS yaml config [748](https://github.com/ethyca/fidesops/pull/748)
 * Adds endpoint to get available connectors (database and saas) [#768](https://github.com/ethyca/fidesops/pull/768)
+* Adds endpoint to get the secrets required for different connectors [#795](https://github.com/ethyca/fidesops/pull/795)
 
 ### Fixed
 * Resolve issue with MyPy seeing files in fidesops as missing imports [#719](https://github.com/ethyca/fidesops/pull/719)

docs/fidesops/docs/guides/connection_types.md

@@ -3,7 +3,7 @@
 
 ## Available Connection Types
 
-To view a list of all available connection types, visit `/api/v1/connection_type`.
+To view a list of all available connection types, visit `GET /api/v1/connection_type`.
 This endpoint can be filtered with a `search` query param and is subject to change.  We include
 database options and third party API services with which Fidesops can communicate.
 
@@ -31,3 +31,31 @@ database options and third party API services with which Fidesops can communicat
     "size": 50
 }
 ```
+
+## Required Connection Secrets
+
+To view the secrets needed to authenticate with a given connection, visit `GET /api/v1/connection_type/<connection_type>/secret`.
+
+### Example
+```json title="<code>GET /api/v1/connection_type/sentry/secret</code>"
+{
+    "title": "sentry_connector_schema",
+    "description": "Sentry secrets schema",
+    "type": "object",
+    "properties": {
+        "access_token": {
+            "title": "Access Token",
+            "type": "string"
+        },
+        "domain": {
+            "title": "Domain",
+            "default": "sentry.io",
+            "type": "string"
+        }
+    },
+    "required": [
+        "access_token"
+    ],
+    "additionalProperties": false
+}
+```

docs/fidesops/docs/postman/Fidesops.postman_collection.json

@@ -3871,6 +3871,35 @@
 						}
 					},
 					"response": []
+				},
+				{
+					"name": "Get connection secrets schema",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{host}}/connection_type/outreach/secret",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"connection_type",
+								"outreach",
+								"secret"
+							]
+						}
+					},
+					"response": []
 				}
 			]
 		}

src/fidesops/api/v1/endpoints/connection_type_endpoints.py

@@ -1,32 +1,33 @@
 import logging
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException
 from fastapi.params import Security
 from fastapi_pagination import Page, Params, paginate
 from fastapi_pagination.bases import AbstractPage
+from starlette.status import HTTP_404_NOT_FOUND
 
 from fidesops.api.v1.scope_registry import CONNECTION_TYPE_READ
-from fidesops.api.v1.urn_registry import CONNECTION_TYPES, V1_URL_PREFIX
+from fidesops.api.v1.urn_registry import (
+    CONNECTION_TYPE_SECRETS,
+    CONNECTION_TYPES,
+    V1_URL_PREFIX,
+)
 from fidesops.models.connectionconfig import ConnectionType
-from fidesops.schemas.saas.saas_config import SaaSType
+from fidesops.schemas.connection_configuration import (
+    SaaSSchemaFactory,
+    secrets_validators,
+)
+from fidesops.schemas.saas.saas_config import SaaSConfig, SaaSType
 from fidesops.util.oauth_util import verify_oauth_client
+from fidesops.util.saas_util import load_config
 
 router = APIRouter(tags=["Connection Types"], prefix=V1_URL_PREFIX)
 
 logger = logging.getLogger(__name__)
 
 
-@router.get(
-    CONNECTION_TYPES,
-    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_TYPE_READ])],
-    response_model=Page[str],
-)
-def get_all_connection_types(
-    *, params: Params = Depends(), search: Optional[str] = None
-) -> AbstractPage[str]:
-    """Returns a list of connection options in Fidesops - includes only database and saas options here."""
-
+def get_connection_types(search: Optional[str] = None) -> List[str]:
     def is_match(elem: str) -> bool:
         """If a search query param was included, is it a substring of an available connector type?"""
         return search in elem if search else True
@@ -45,4 +46,47 @@ def is_match(elem: str) -> bool:
     ]
     connection_types: List[str] = sorted(database_types + saas_types)
 
+    return connection_types
+
+
+@router.get(
+    CONNECTION_TYPES,
+    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_TYPE_READ])],
+    response_model=Page[str],
+)
+def get_all_connection_types(
+    *, params: Params = Depends(), search: Optional[str] = None
+) -> AbstractPage[str]:
+    """Returns a list of connection options in Fidesops - includes only database and saas options here."""
+
+    connection_types: List[str] = get_connection_types(search)
+
     return paginate(connection_types, params)
+
+
+@router.get(
+    CONNECTION_TYPE_SECRETS,
+    dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_TYPE_READ])],
+)
+def get_connection_type_secret_schema(
+    *, connection_type: str
+) -> Optional[Dict[str, Any]]:
+    """Returns the secret fields that should be supplied to authenticate with a particular connection type
+
+    Note that this endpoint should never return actual secrets, we return the *types* of secret fields needed
+    to authenticate.
+    """
+    connection_types: List[str] = get_connection_types()
+    if connection_type not in connection_types:
+        raise HTTPException(
+            status_code=HTTP_404_NOT_FOUND,
+            detail=f"No connection type found with name '{connection_type}'.",
+        )
+
+    if connection_type in [db_type.value for db_type in ConnectionType]:
+        return secrets_validators[connection_type].schema()
+
+    config: SaaSConfig = SaaSConfig(
+        **load_config(f"data/saas/config/{connection_type}_config.yml")
+    )
+    return SaaSSchemaFactory(config).get_saas_schema().schema()

src/fidesops/api/v1/urn_registry.py

@@ -63,7 +63,7 @@
 
 # Connection Type URLs
 CONNECTION_TYPES = "/connection_type"
-
+CONNECTION_TYPE_SECRETS = "/connection_type/{connection_type}/secret"
 
 # Connection Configurations URLs
 CONNECTIONS = "/connection"

src/fidesops/schemas/connection_configuration/connection_secrets_saas.py

@@ -58,8 +58,11 @@ def get_saas_schema(self) -> Type[SaaSSchema]:
                 if connector_param.default_value
                 else (str, ...)
             )
-        return create_model(
+        SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema"  # Dynamically override the docstring
+        model: Type[SaaSSchema] = create_model(
             f"{self.saas_config.fides_key}_schema",
             **field_definitions,
             __base__=SaaSSchema,
         )
+
+        return model

src/fidesops/util/saas_util.py

@@ -7,6 +7,8 @@
 from functools import reduce
 from typing import Any, Dict, List, Optional, Set, Tuple
 
+import yaml
+from fideslib.core.config import load_file
 from multidimensional_urlencode import urlencode as multidimensional_urlencode
 
 from fidesops.common_exceptions import FidesopsException
@@ -19,6 +21,13 @@
 FIDESOPS_GROUPED_INPUTS = "fidesops_grouped_inputs"
 
 
+def load_config(filename: str) -> Dict:
+    """Loads the saas config from the yaml file"""
+    yaml_file = load_file([filename])
+    with open(yaml_file, "r") as file:
+        return yaml.safe_load(file).get("saas_config", [])
+
+
 def merge_fields(target: Field, source: Field) -> Field:
     """Replaces source references and identities if they are available from the target"""
     if source.references is not None:

tests/api/v1/endpoints/test_connection_template_endpoints.py

@@ -4,7 +4,11 @@
 from starlette.testclient import TestClient
 
 from fidesops.api.v1.scope_registry import CONNECTION_READ, CONNECTION_TYPE_READ
-from fidesops.api.v1.urn_registry import CONNECTION_TYPES, V1_URL_PREFIX
+from fidesops.api.v1.urn_registry import (
+    CONNECTION_TYPE_SECRETS,
+    CONNECTION_TYPES,
+    V1_URL_PREFIX,
+)
 
 
 class TestGetConnections:
@@ -57,3 +61,82 @@ def test_search_connection_types(self, api_client, generate_auth_header, url):
         data = resp.json()
         assert data["total"] == 3
         assert data["items"] == ["outreach", "postgres", "redshift"]
+
+
+class TestGetConnectionSecretSchema:
+    @pytest.fixture(scope="function")
+    def base_url(self, oauth_client: ClientDetail, policy) -> str:
+        return V1_URL_PREFIX + CONNECTION_TYPE_SECRETS
+
+    def test_get_connection_secret_schema_not_authenticated(self, api_client, base_url):
+        resp = api_client.get(base_url.format(connection_type="sentry"), headers={})
+        assert resp.status_code == 401
+
+    def test_get_connection_secret_schema_forbidden(
+        self, api_client, base_url, generate_auth_header
+    ):
+        auth_header = generate_auth_header(scopes=[CONNECTION_READ])
+        resp = api_client.get(
+            base_url.format(connection_type="sentry"), headers=auth_header
+        )
+        assert resp.status_code == 403
+
+    def test_get_connection_secret_schema_not_found(
+        self, api_client: TestClient, generate_auth_header, base_url
+    ):
+        auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ])
+        resp = api_client.get(
+            base_url.format(connection_type="connection_type_we_do_not_support"),
+            headers=auth_header,
+        )
+        assert resp.status_code == 404
+        assert (
+            resp.json()["detail"]
+            == "No connection type found with name 'connection_type_we_do_not_support'."
+        )
+
+    def test_get_connection_secret_schema_mongodb(
+        self, api_client: TestClient, generate_auth_header, base_url
+    ) -> None:
+        auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ])
+        resp = api_client.get(
+            base_url.format(connection_type="mongodb"), headers=auth_header
+        )
+        assert resp.json() == {
+            "title": "MongoDBSchema",
+            "description": "Schema to validate the secrets needed to connect to a MongoDB Database",
+            "type": "object",
+            "properties": {
+                "url": {"title": "Url", "type": "string"},
+                "username": {"title": "Username", "type": "string"},
+                "password": {"title": "Password", "type": "string"},
+                "host": {"title": "Host", "type": "string"},
+                "port": {"title": "Port", "type": "integer"},
+                "defaultauthdb": {"title": "Defaultauthdb", "type": "string"},
+            },
+            "additionalProperties": False,
+        }
+
+    def test_get_connection_secret_schema_hubspot(
+        self, api_client: TestClient, generate_auth_header, base_url
+    ) -> None:
+        auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ])
+        resp = api_client.get(
+            base_url.format(connection_type="hubspot"), headers=auth_header
+        )
+
+        assert resp.json() == {
+            "title": "hubspot_connector_example_schema",
+            "description": "Hubspot secrets schema",
+            "type": "object",
+            "properties": {
+                "hapikey": {"title": "Hapikey", "type": "string"},
+                "domain": {
+                    "title": "Domain",
+                    "default": "api.hubapi.com",
+                    "type": "string",
+                },
+            },
+            "required": ["hapikey"],
+            "additionalProperties": False,
+        }

tests/fixtures/saas/hubspot_fixtures.py

@@ -16,9 +16,8 @@
 from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
 from fidesops.service.connectors import SaaSConnector
 from fidesops.util import cryptographic_util
-from fidesops.util.saas_util import format_body
+from fidesops.util.saas_util import format_body, load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 from tests.test_helpers.saas_test_utils import poll_for_existence
 
 saas_config = load_toml(["saas_config.toml"])

tests/fixtures/saas/mailchimp_fixtures.py

@@ -16,8 +16,8 @@
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
 from fidesops.service.connectors.saas_connector import SaaSConnector
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml(["saas_config.toml"])
 

tests/fixtures/saas/outreach_fixtures.py

@@ -15,8 +15,8 @@
 )
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.util import cryptographic_util
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml(["saas_config.toml"])
 

tests/fixtures/saas/salesforce_fixtures.py

@@ -15,8 +15,8 @@
 )
 from fidesops.models.datasetconfig import DatasetConfig
 from fidesops.util import cryptographic_util
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml(["saas_config.toml"])
 

tests/fixtures/saas/segment_fixtures.py

@@ -17,8 +17,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 from tests.test_helpers.saas_test_utils import poll_for_existence
 
 saas_config = load_toml(["saas_config.toml"])

tests/fixtures/saas/sentry_fixtures.py

@@ -13,8 +13,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml(["saas_config.toml"])
 

tests/fixtures/saas/stripe_fixtures.py

@@ -15,8 +15,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
+from fidesops.util.saas_util import load_config
 from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml(["saas_config.toml"])