diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c2c3749..6ac26ebde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,8 @@ The types of changes are: * Added erasure endpoints for Shopify connector [#1289](https://github.com/ethyca/fidesops/pull/1289) * Adds ability to send email notification upon privacy request completion [#1282](https://github.com/ethyca/fidesops/pull/1282) +* Added human readable label to ConnectionType endpoint [#1297](https://github.com/ethyca/fidesops/pull/1297) + ### Docs * Fix analytics opt out environment variable name [#1170](https://github.com/ethyca/fidesops/pull/1170) diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml index 9ecc15e28..b5187baf3 100644 --- a/data/saas/saas_connector_registry.toml +++ b/data/saas/saas_connector_registry.toml @@ -2,68 +2,83 @@ config = "data/saas/config/adobe_campaign_config.yml" dataset = "data/saas/dataset/adobe_campaign_dataset.yml" icon = "data/saas/icon/adobe.svg" +human_readable = "Adobe Campaign" [auth0] config = "data/saas/config/auth0_config.yml" dataset = "data/saas/dataset/auth0_dataset.yml" icon = "data/saas/icon/default.svg" +human_readable = "Auth0" [datadog] config = "data/saas/config/datadog_config.yml" dataset = "data/saas/dataset/datadog_dataset.yml" icon = "data/saas/icon/default.svg" +human_readable = "Datadog" [hubspot] config = "data/saas/config/hubspot_config.yml" dataset = "data/saas/dataset/hubspot_dataset.yml" icon = "data/saas/icon/hubspot.svg" +human_readable = "HubSpot" + [logi_id] config = "data/saas/config/logi_id_config.yml" dataset = "data/saas/dataset/logi_id_dataset.yml" icon = "data/saas/icon/default.svg" +human_readable = "Logi ID" [mailchimp] config = "data/saas/config/mailchimp_config.yml" dataset = "data/saas/dataset/mailchimp_dataset.yml" icon = "data/saas/icon/mailchimp.svg" +human_readable = "Mailchimp" [outreach] config = "data/saas/config/outreach_config.yml" dataset = "data/saas/dataset/outreach_dataset.yml" icon = "data/saas/icon/outreach.svg" +human_readable = "Outreach" [salesforce] config = "data/saas/config/salesforce_config.yml" dataset = "data/saas/dataset/salesforce_dataset.yml" icon = "data/saas/icon/salesforce.svg" +human_readable = "Salesforce" [segment] config = "data/saas/config/segment_config.yml" dataset = "data/saas/dataset/segment_dataset.yml" icon = "data/saas/icon/segment.svg" +human_readable = "Segment" [sendgrid] config = "data/saas/config/sendgrid_config.yml" dataset = "data/saas/dataset/sendgrid_dataset.yml" icon = "data/saas/icon/default.svg" +human_readable = "SendGrid" [sentry] config = "data/saas/config/sentry_config.yml" dataset = "data/saas/dataset/sentry_dataset.yml" icon = "data/saas/icon/sentry.svg" +human_readable = "Sentry" [shopify] config = "data/saas/config/shopify_config.yml" dataset = "data/saas/dataset/shopify_dataset.yml" icon = "data/saas/icon/default.svg" +human_readable = "Shopify" [stripe] config = "data/saas/config/stripe_config.yml" dataset = "data/saas/dataset/stripe_dataset.yml" icon = "data/saas/icon/stripe.svg" +human_readable = "Stripe" [zendesk] config = "data/saas/config/zendesk_config.yml" dataset = "data/saas/dataset/zendesk_dataset.yml" -icon = "data/saas/icon/zendesk.svg" \ No newline at end of file +icon = "data/saas/icon/zendesk.svg" +human_readable = "Zendesk" diff --git a/docs/fidesops/docs/guides/connection_types.md b/docs/fidesops/docs/guides/connection_types.md index 9d411d333..9e08ff8e0 100644 --- a/docs/fidesops/docs/guides/connection_types.md +++ b/docs/fidesops/docs/guides/connection_types.md @@ -4,98 +4,130 @@ ## Available Connection Types To view a list of all available connection types, visit `GET /api/v1/connection_type`. -This endpoint can be filtered with a `search` query param and is subject to change. We include -database options and third party API services with which fidesops can communicate. +This endpoint can be filtered with a `search` query param or a `system_type` query param +and is subject to change. We include database options and third party API services with +which fidesops can communicate. ```json title="GET /api/v1/connection_type" { "items": [ { "identifier": "bigquery", - "type": "database" + "type": "database", + "human_readable": "BigQuery" }, { "identifier": "mariadb", - "type": "database" + "type": "database", + "human_readable": "MariaDB" }, { "identifier": "mongodb", - "type": "database" + "type": "database", + "human_readable": "MongoDB" }, { "identifier": "mssql", - "type": "database" + "type": "database", + "human_readable": "Microsoft SQL Server" }, { "identifier": "mysql", - "type": "database" + "type": "database", + "human_readable": "MySQL" }, { "identifier": "postgres", - "type": "database" + "type": "database", + "human_readable": "PostgreSQL" }, { "identifier": "redshift", - "type": "database" + "type": "database", + "human_readable": "Amazon Redshift" }, { "identifier": "snowflake", - "type": "database" + "type": "database", + "human_readable": "Snowflake" }, { "identifier": "adobe_campaign", - "type": "saas" + "type": "saas", + "human_readable": "Adobe Campaign" }, { "identifier": "auth0", - "type": "saas" + "type": "saas", + "human_readable": "Auth0" }, { "identifier": "datadog", - "type": "saas" + "type": "saas", + "human_readable": "Datadog" }, { "identifier": "hubspot", - "type": "saas" + "type": "saas", + "human_readable": "HubSpot" }, { "identifier": "logi_id", - "type": "saas" + "type": "saas", + "human_readable": "Logi ID" }, { "identifier": "mailchimp", - "type": "saas" + "type": "saas", + "human_readable": "Mailchimp" }, { "identifier": "outreach", - "type": "saas" + "type": "saas", + "human_readable": "Outreach" }, { "identifier": "salesforce", - "type": "saas" + "type": "saas", + "human_readable": "Salesforce" }, { "identifier": "segment", - "type": "saas" + "type": "saas", + "human_readable": "Segment" }, { "identifier": "sendgrid", - "type": "saas" + "type": "saas", + "human_readable": "SendGrid" }, { "identifier": "sentry", - "type": "saas" + "type": "saas", + "human_readable": "Sentry" + }, + { + "identifier": "shopify", + "type": "saas", + "human_readable": "Shopify" }, { "identifier": "stripe", - "type": "saas" + "type": "saas", + "human_readable": "Stripe" }, { "identifier": "zendesk", - "type": "saas" + "type": "saas", + "human_readable": "Zendesk" + }, + { + "identifier": "manual_webhook", + "type": "manual", + "human_readable": "Manual Webhook" } ], - "total": 21, + "total": 23, "page": 1, "size": 50 } diff --git a/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py index 49cd6eaa9..9391f0ecd 100644 --- a/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/connection_type_endpoints.py @@ -23,6 +23,11 @@ SystemType, ) from fidesops.ops.schemas.saas.saas_config import SaaSConfig, SaaSType +from fidesops.ops.service.connectors.saas.connector_registry_service import ( + ConnectorRegistry, + load_registry, + registry_file, +) from fidesops.ops.util.oauth_util import verify_oauth_client from fidesops.ops.util.saas_util import load_config @@ -57,7 +62,11 @@ def is_match(elem: str) -> bool: ) connection_system_types.extend( [ - ConnectionSystemTypeMap(identifier=item, type=SystemType.database) + ConnectionSystemTypeMap( + identifier=item, + type=SystemType.database, + human_readable=ConnectionType(item).human_readable, + ) for item in database_types ] ) @@ -69,12 +78,22 @@ def is_match(elem: str) -> bool: if saas_type != SaaSType.custom and is_match(saas_type.value) ] ) - connection_system_types.extend( - [ - ConnectionSystemTypeMap(identifier=item, type=SystemType.saas) - for item in saas_types - ] - ) + registry: ConnectorRegistry = load_registry(registry_file) + + for item in saas_types: + human_readable_name: str = item + if registry.get_connector_template(item) is not None: + human_readable_name = registry.get_connector_template( # type: ignore[union-attr] + item + ).human_readable + + connection_system_types.append( + ConnectionSystemTypeMap( + identifier=item, + type=SystemType.saas, + human_readable=human_readable_name, + ) + ) if system_type == SystemType.manual or system_type is None: manual_types: List[str] = sorted( @@ -87,7 +106,11 @@ def is_match(elem: str) -> bool: ) connection_system_types.extend( [ - ConnectionSystemTypeMap(identifier=item, type=SystemType.manual) + ConnectionSystemTypeMap( + identifier=item, + type=SystemType.manual, + human_readable=ConnectionType(item).human_readable, + ) for item in manual_types ] ) diff --git a/src/fidesops/ops/models/connectionconfig.py b/src/fidesops/ops/models/connectionconfig.py index fbd71c7b3..23dab4aa1 100644 --- a/src/fidesops/ops/models/connectionconfig.py +++ b/src/fidesops/ops/models/connectionconfig.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import Any, Optional, Type +from typing import Any, Dict, Optional, Type from fideslib.db.base import Base from fideslib.db.base_class import get_key_from_data @@ -48,6 +48,33 @@ class ConnectionType(enum.Enum): email = "email" manual_webhook = "manual_webhook" # Run before the traversal + @property + def human_readable(self) -> str: + """Human-readable mapping for ConnectionTypes + Add to this mapping if you add a new ConnectionType + """ + readable_mapping: Dict[str, str] = { + ConnectionType.postgres.value: "PostgreSQL", + ConnectionType.mongodb.value: "MongoDB", + ConnectionType.mysql.value: "MySQL", + ConnectionType.https.value: "Policy Webhook", + ConnectionType.saas.value: "SaaS", + ConnectionType.redshift.value: "Amazon Redshift", + ConnectionType.snowflake.value: "Snowflake", + ConnectionType.mssql.value: "Microsoft SQL Server", + ConnectionType.mariadb.value: "MariaDB", + ConnectionType.bigquery.value: "BigQuery", + ConnectionType.manual.value: "Manual Connector", + ConnectionType.email.value: "Email Connector", + ConnectionType.manual_webhook.value: "Manual Webhook", + } + try: + return readable_mapping[self.value] + except KeyError: + raise NotImplementedError( + "Add new ConnectionType to human_readable mapping" + ) + class AccessLevel(enum.Enum): """ diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_config.py b/src/fidesops/ops/schemas/connection_configuration/connection_config.py index 3ca51d05d..c3e85c9ad 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_config.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_config.py @@ -62,6 +62,7 @@ class ConnectionSystemTypeMap(BaseModel): identifier: Union[ConnectionType, SaaSType] type: SystemType + human_readable: str class Config: """Use enum values and set orm mode""" diff --git a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py index 6d745dffa..761bcfe51 100644 --- a/src/fidesops/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fidesops/ops/service/connectors/saas/connector_registry_service.py @@ -38,6 +38,7 @@ class ConnectorTemplate(BaseModel): config: str dataset: str icon: str + human_readable: str @validator("config") def validate_config(cls, config: str) -> str: diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 36be16892..d6d9bb23a 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -21,10 +21,7 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.schemas.connection_configuration.connection_config import ( - ConnectionSystemTypeMap, - SystemType, -) +from fidesops.ops.schemas.connection_configuration.connection_config import SystemType from fidesops.ops.schemas.saas.saas_config import SaaSType @@ -56,10 +53,12 @@ def test_get_connection_types( assert { "identifier": ConnectionType.postgres.value, "type": SystemType.database.value, + "human_readable": "PostgreSQL", } in data assert { "identifier": SaaSType.stripe.value, "type": SystemType.saas.value, + "human_readable": "Stripe", } in data assert "saas" not in [item["identifier"] for item in data] @@ -77,6 +76,7 @@ def test_search_connection_types(self, api_client, generate_auth_header, url): assert data[0] == { "identifier": SaaSType.stripe.value, "type": SystemType.saas.value, + "human_readable": "Stripe", } resp = api_client.get(url + "?search=re", headers=auth_header) @@ -87,12 +87,18 @@ def test_search_connection_types(self, api_client, generate_auth_header, url): { "identifier": ConnectionType.postgres.value, "type": SystemType.database.value, + "human_readable": "PostgreSQL", }, { "identifier": ConnectionType.redshift.value, "type": SystemType.database.value, + "human_readable": "Amazon Redshift", + }, + { + "identifier": SaaSType.outreach.value, + "type": SystemType.saas.value, + "human_readable": "Outreach", }, - {"identifier": SaaSType.outreach.value, "type": SystemType.saas.value}, ] def test_search_connection_types_case_insensitive( @@ -107,10 +113,12 @@ def test_search_connection_types_case_insensitive( assert data[0] == { "identifier": ConnectionType.postgres.value, "type": SystemType.database.value, + "human_readable": "PostgreSQL", } assert data[1] == { "identifier": SaaSType.stripe.value, "type": SystemType.saas.value, + "human_readable": "Stripe", } resp = api_client.get(url + "?search=Re", headers=auth_header) @@ -121,12 +129,18 @@ def test_search_connection_types_case_insensitive( { "identifier": ConnectionType.postgres.value, "type": SystemType.database.value, + "human_readable": "PostgreSQL", }, { "identifier": ConnectionType.redshift.value, "type": SystemType.database.value, + "human_readable": "Amazon Redshift", + }, + { + "identifier": SaaSType.outreach.value, + "type": SystemType.saas.value, + "human_readable": "Outreach", }, - {"identifier": SaaSType.outreach.value, "type": SystemType.saas.value}, ] def test_search_system_type(self, api_client, generate_auth_header, url): @@ -169,7 +183,13 @@ def test_search_manual_system_type(self, api_client, generate_auth_header, url): assert resp.status_code == 200 data = resp.json()["items"] assert len(data) == 1 - assert data == [{"identifier": "manual_webhook", "type": "manual"}] + assert data == [ + { + "identifier": "manual_webhook", + "type": "manual", + "human_readable": "Manual Webhook", + } + ] class TestGetConnectionSecretSchema: diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index f57124c93..5a0836b90 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -146,3 +146,11 @@ def test_default_value_saas_config_invalid_type( saas_example_config["type"] = "invalid" with pytest.raises(ValueError): SaaSConfig(**saas_example_config) + + def test_connection_type_human_readable(self): + for connection in ConnectionType: + connection.human_readable # Makes sure all ConnectionTypes have been added to human_readable mapping + + def test_connection_type_human_readable_invalid(self): + with pytest.raises(ValueError): + ConnectionType("nonmapped_type").human_readable() diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index 8c482ac82..26d0d85f2 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -18,4 +18,5 @@ def test_get_connector_template(self): config="data/saas/config/mailchimp_config.yml", dataset="data/saas/dataset/mailchimp_dataset.yml", icon="data/saas/icon/mailchimp.svg", + human_readable="Mailchimp", )