New integration options for D&D (#5000)

ethyca · Jun 19, 2024 · a48183e · a48183e
1 parent 39d23e1
commit a48183e
Show file tree

Hide file tree

Showing 82 changed files with 1,695 additions and 512 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,7 @@ The types of changes are:
 - Implement sending emails via property-specific messaging templates [#4950](https://github.com/ethyca/fides/pull/4950)
 - New privacy request search to replace existing endpoint [#4987](https://github.com/ethyca/fides/pull/4987)
 - Added new Google Cloud SQL for MySQL Connector [#4949](https://github.com/ethyca/fides/pull/4949)
+- Add new options for integrations for discovery & detection [#5000](https://github.com/ethyca/fides/pull/5000)
 
 ### Changed
 - Move new data map reporting table out of beta and remove old table from Data Lineage map. [#4963](https://github.com/ethyca/fides/pull/4963)

diff --git a/clients/admin-ui/cypress/e2e/integration-management.cy.ts b/clients/admin-ui/cypress/e2e/integration-management.cy.ts
@@ -11,6 +11,10 @@ describe("Integration management for data detection & discovery", () => {
         test_status: "succeeded",
       },
     }).as("testConnection");
+
+    cy.intercept("GET", "/api/v1/connection_type/*/secret", {
+      fixture: "connectors/bigquery_secret.json",
+    }).as("getSecretsSchema");
   });
 
   describe("accessing the page", () => {
@@ -33,7 +37,7 @@ describe("Integration management for data detection & discovery", () => {
     });
 
     it("should show an empty state when there are no integrations available", () => {
-      cy.intercept("GET", "/api/v1/connection*", {
+      cy.intercept("GET", "/api/v1/connection?*", {
         fixture: "connectors/empty_list.json",
       }).as("getConnections");
       cy.visit(INTEGRATION_MANAGEMENT_ROUTE);
@@ -43,7 +47,7 @@ describe("Integration management for data detection & discovery", () => {
 
     describe("list view", () => {
       beforeEach(() => {
-        cy.intercept("GET", "/api/v1/connection*", {
+        cy.intercept("GET", "/api/v1/connection?*", {
           fixture: "connectors/bigquery_connection_list.json",
         }).as("getConnections");
         cy.visit(INTEGRATION_MANAGEMENT_ROUTE);
@@ -72,9 +76,12 @@ describe("Integration management for data detection & discovery", () => {
 
     describe("adding an integration", () => {
       beforeEach(() => {
-        cy.intercept("GET", "/api/v1/connection*", {
+        cy.intercept("GET", "/api/v1/connection?*", {
           fixture: "connectors/bigquery_connection_list.json",
         }).as("getConnections");
+        cy.intercept("GET", "/api/v1/connection_type", {
+          fixture: "connectors/connection_types.json",
+        }).as("getConnectionTypes");
         cy.visit(INTEGRATION_MANAGEMENT_ROUTE);
         cy.wait("@getConnections");
       });
@@ -88,31 +95,24 @@ describe("Integration management for data detection & discovery", () => {
           });
       });
 
-      it("should be able to add a new BigQuery integration", () => {
-        cy.intercept("PATCH", "/api/v1/connection").as("patchConnection");
-        cy.getByTestId("add-integration-btn").click();
-        cy.getByTestId("add-modal-content").within(() => {
-          cy.getByTestId("configure-btn").click();
-        });
-        cy.getByTestId("input-name").type("test name");
-        cy.getByTestId("input-description").type("test description");
-        cy.getByTestId("save-btn").click();
-        cy.wait("@patchConnection");
-      });
-
       it("should be able to add a new integration with secrets", () => {
         cy.intercept("PATCH", "/api/v1/connection").as("patchConnection");
         cy.intercept("PUT", "/api/v1/connection/*/secret*").as(
           "putConnectionSecrets"
         );
         cy.getByTestId("add-integration-btn").click();
         cy.getByTestId("add-modal-content").within(() => {
-          cy.getByTestId("configure-btn").click();
+          cy.getByTestId("integration-info-bq_placeholder").within(() => {
+            cy.getByTestId("configure-btn").click();
+          });
         });
         cy.getByTestId("input-name").type("test name");
-        cy.getByTestId("input-keyfile_creds").type(`{"credentials": "test"}`, {
-          parseSpecialCharSequences: false,
-        });
+        cy.getByTestId("input-secrets.keyfile_creds").type(
+          `{"credentials": "test"}`,
+          {
+            parseSpecialCharSequences: false,
+          }
+        );
         cy.getByTestId("save-btn").click();
         cy.wait("@patchConnection");
         cy.wait("@putConnectionSecrets");
@@ -128,9 +128,17 @@ describe("Integration management for data detection & discovery", () => {
         }).as("getSystems");
         cy.getByTestId("add-integration-btn").click();
         cy.getByTestId("add-modal-content").within(() => {
-          cy.getByTestId("configure-btn").click();
+          cy.getByTestId("integration-info-bq_placeholder").within(() => {
+            cy.getByTestId("configure-btn").click();
+          });
         });
         cy.getByTestId("input-name").type("test name");
+        cy.getByTestId("input-secrets.keyfile_creds").type(
+          `{"credentials": "test"}`,
+          {
+            parseSpecialCharSequences: false,
+          }
+        );
         cy.selectOption("input-system_fides_key", "Fidesctl System");
         cy.getByTestId("save-btn").click();
         cy.wait("@patchSystemConnection");
@@ -144,9 +152,19 @@ describe("Integration management for data detection & discovery", () => {
       cy.intercept("GET", "/api/v1/connection/*", {
         fixture: "connectors/bigquery_connection.json",
       }).as("getConnection");
+      cy.intercept("GET", "/api/v1/connection_type", {
+        fixture: "connectors/connection_types.json",
+      }).as("getConnectionTypes");
       cy.visit("/integrations/bq_integration");
     });
 
+    it("redirects to list view if the integration type is incorrect", () => {
+      cy.intercept("GET", "/api/v1/connection/*", {
+        fixture: "connectors/postgres_connector.json",
+      }).as("getConnection");
+      cy.url().should("not.contain", "bq_integration");
+    });
+
     it("can test the connection", () => {
       cy.getByTestId("test-connection-btn").click();
       cy.wait("@testConnection");
@@ -160,6 +178,12 @@ describe("Integration management for data detection & discovery", () => {
         .should("have.value", "BQ Integration")
         .clear()
         .type("A different name");
+      cy.getByTestId("input-secrets.keyfile_creds").type(
+        `{"credentials": "test"}`,
+        {
+          parseSpecialCharSequences: false,
+        }
+      );
       cy.getByTestId("save-btn").click();
       cy.wait("@patchConnection");
     });

diff --git a/clients/admin-ui/cypress/fixtures/connectors/bigquery_secret.json b/clients/admin-ui/cypress/fixtures/connectors/bigquery_secret.json
@@ -0,0 +1,23 @@
+{
+  "title": "BigQuerySchema",
+  "description": "Schema to validate the secrets needed to connect to BigQuery",
+  "type": "object",
+  "properties": {
+    "keyfile_creds": {
+      "title": "Keyfile Creds",
+      "description": "The contents of the key file that contains authentication credentials for a service account in GCP.",
+      "sensitive": true,
+      "allOf": [
+        {
+          "$ref": "#/definitions/KeyfileCreds"
+        }
+      ]
+    },
+    "dataset": {
+      "title": "BigQuery Dataset",
+      "description": "The dataset within your BigQuery project that contains the tables you want to access.",
+      "type": "string"
+    }
+  },
+  "required": ["keyfile_creds"]
+}
diff --git a/clients/admin-ui/src/features/common/FidesSpinner.tsx b/clients/admin-ui/src/features/common/FidesSpinner.tsx
@@ -0,0 +1,9 @@
+import { Flex, Spinner, SpinnerProps } from "fidesui";
+
+const FidesSpinner = ({ ...props }: SpinnerProps) => (
+  <Flex boxSize="full" align="center" justify="center">
+    <Spinner color="primary" {...props} />
+  </Flex>
+);
+
+export default FidesSpinner;
diff --git a/clients/admin-ui/src/features/common/copy/ShowMoreContent.tsx b/clients/admin-ui/src/features/common/copy/ShowMoreContent.tsx
@@ -0,0 +1,21 @@
+import { Collapse, Text, useDisclosure } from "fidesui";
+import { ReactNode } from "react";
+
+const ShowMoreContent = ({ children }: { children: ReactNode }) => {
+  const { isOpen, onToggle } = useDisclosure();
+  return (
+    <>
+      <Collapse in={isOpen}>{children}</Collapse>
+      <Text
+        fontSize="sm"
+        cursor="pointer"
+        textDecoration="underline"
+        onClick={onToggle}
+      >
+        {isOpen ? "Show less" : "Show more"}
+      </Text>
+    </>
+  );
+};
+
+export default ShowMoreContent;
diff --git a/clients/admin-ui/src/features/common/copy/components.tsx b/clients/admin-ui/src/features/common/copy/components.tsx
@@ -0,0 +1,38 @@
+import { Heading, Link, OrderedList, Text, UnorderedList } from "fidesui";
+import { ReactNode } from "react";
+
+export const InfoHeading = ({ text }: { text: string }) => (
+  <Heading fontSize="sm" mt={4} mb={1}>
+    {text}
+  </Heading>
+);
+
+export const InfoText = ({ children }: { children: ReactNode }) => (
+  <Text fontSize="14px" mb={4}>
+    {children}
+  </Text>
+);
+
+export const InfoLink = ({
+  children,
+  href,
+}: {
+  children: ReactNode;
+  href: string;
+}) => (
+  <Link href={href} textDecoration="underline" isExternal>
+    {children}
+  </Link>
+);
+
+export const InfoUnorderedList = ({ children }: { children: ReactNode }) => (
+  <UnorderedList fontSize="14px" mb={4}>
+    {children}
+  </UnorderedList>
+);
+
+export const InfoOrderedList = ({ children }: { children: ReactNode }) => (
+  <OrderedList fontSize="14px" mb={4}>
+    {children}
+  </OrderedList>
+);
diff --git a/clients/admin-ui/src/features/connection-type/connection-type.slice.ts b/clients/admin-ui/src/features/connection-type/connection-type.slice.ts
@@ -14,7 +14,7 @@ import {
 import type { RootState } from "../../app/store";
 import {
   ConnectionTypeParams,
-  ConnectionTypeSecretSchemaReponse,
+  ConnectionTypeSecretSchemaResponse,
   ConnectionTypeState,
 } from "./types";
 
@@ -118,7 +118,7 @@ export const connectionTypeApi = baseApi.injectEndpoints({
       providesTags: () => ["Connection Type"],
     }),
     getConnectionTypeSecretSchema: build.query<
-      ConnectionTypeSecretSchemaReponse,
+      ConnectionTypeSecretSchemaResponse,
       string
     >({
       query: (connectionType) => ({

diff --git a/clients/admin-ui/src/features/connection-type/types.ts b/clients/admin-ui/src/features/connection-type/types.ts
@@ -23,16 +23,16 @@ export type ConnectionTypeSecretSchemaProperty = {
   sensitive?: boolean;
 };
 
-export type ConnectionTypeSecretSchemaReponse = {
+export type ConnectionTypeSecretSchemaResponse = {
   additionalProperties: boolean;
   description: string;
   properties: { [key: string]: ConnectionTypeSecretSchemaProperty };
   required: string[];
   title: string;
   type: string;
   definitions: {
-    ExtendedIdentityTypes: ConnectionTypeSecretSchemaReponse;
-    AdvancedSettingsWithExtendedIdentityTypes: ConnectionTypeSecretSchemaReponse;
+    ExtendedIdentityTypes: ConnectionTypeSecretSchemaResponse;
+    AdvancedSettingsWithExtendedIdentityTypes: ConnectionTypeSecretSchemaResponse;
   };
 };
 

diff --git a/...min-ui/src/features/datastore-connections/add-connection/database/ConnectorParameters.tsx b/...min-ui/src/features/datastore-connections/add-connection/database/ConnectorParameters.tsx
@@ -4,21 +4,22 @@ import {
   selectConnectionTypeState,
   setConnection,
 } from "connection-type/connection-type.slice";
-import { ConnectionTypeSecretSchemaReponse } from "connection-type/types";
+import { ConnectionTypeSecretSchemaResponse } from "connection-type/types";
 import {
   usePatchDatastoreConnectionMutation,
   useUpdateDatastoreConnectionSecretsMutation,
 } from "datastore-connections/datastore-connection.slice";
-import {
-  DatastoreConnectionRequest,
-  DatastoreConnectionSecretsRequest,
-} from "datastore-connections/types";
+import { DatastoreConnectionSecretsRequest } from "datastore-connections/types";
 import { Box } from "fidesui";
 import { useState } from "react";
 import { useDispatch } from "react-redux";
 
 import { useAppSelector } from "~/app/hooks";
-import { ConnectionType } from "~/types/api";
+import {
+  AccessLevel,
+  ConnectionType,
+  CreateConnectionConfigurationWithSecrets,
+} from "~/types/api";
 
 import ConnectorParametersForm from "../forms/ConnectorParametersForm";
 import { formatKey } from "../helpers";
@@ -28,7 +29,7 @@ import {
 } from "../types";
 
 type ConnectorParametersProps = {
-  data: ConnectionTypeSecretSchemaReponse;
+  data: ConnectionTypeSecretSchemaResponse;
   /**
    * Parent callback invoked when a connection is initially created
    */
@@ -60,8 +61,8 @@ export const useDatabaseConnector = ({
   const handleSubmit = async (values: BaseConnectorParametersFields) => {
     try {
       setIsSubmitting(true);
-      const params1: DatastoreConnectionRequest = {
-        access: "write",
+      const params1: CreateConnectionConfigurationWithSecrets = {
+        access: AccessLevel.WRITE,
         connection_type: connectionOption?.identifier as ConnectionType,
         description: values.description,
         disabled: false,

diff --git a/.../admin-ui/src/features/datastore-connections/add-connection/email/ConnectorParameters.tsx b/.../admin-ui/src/features/datastore-connections/add-connection/email/ConnectorParameters.tsx
@@ -1,4 +1,4 @@
-import { ConnectionTypeSecretSchemaReponse } from "connection-type/types";
+import { ConnectionTypeSecretSchemaResponse } from "connection-type/types";
 import { Box } from "fidesui";
 import React from "react";
 
@@ -7,7 +7,7 @@ import ConnectorParametersForm from "~/features/datastore-connections/add-connec
 import { EmailConnectorParametersFormFields } from "~/features/datastore-connections/add-connection/types";
 
 type ConnectorParametersProps = {
-  data: ConnectionTypeSecretSchemaReponse;
+  data: ConnectionTypeSecretSchemaResponse;
   /**
    * Parent callback invoked when a connection is initially created
    */

diff --git a/...in-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx b/...in-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx
@@ -2,7 +2,7 @@ import { useAPIHelper } from "common/hooks";
 import { selectConnectionTypeState } from "connection-type/connection-type.slice";
 import {
   ConnectionTypeSecretSchemaProperty,
-  ConnectionTypeSecretSchemaReponse,
+  ConnectionTypeSecretSchemaResponse,
 } from "connection-type/types";
 import { useLazyGetDatastoreConnectionStatusQuery } from "datastore-connections/datastore-connection.slice";
 import {
@@ -39,7 +39,7 @@ import { fillInDefaults } from "./helpers";
 const FIDES_DATASET_REFERENCE = "#/definitions/FidesDatasetReference";
 
 type ConnectorParametersFormProps = {
-  data: ConnectionTypeSecretSchemaReponse;
+  data: ConnectionTypeSecretSchemaResponse;
   defaultValues:
     | DatabaseConnectorParametersFormFields
     | SaasConnectorParametersFormFields;

diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/helpers.ts b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/helpers.ts
@@ -1,12 +1,12 @@
-import { ConnectionTypeSecretSchemaReponse } from "~/features/connection-type/types";
+import { ConnectionTypeSecretSchemaResponse } from "~/features/connection-type/types";
 
 /**
  * Fill in default values based off of a schema. Use schema defaults if they exist, otherwise use passed-in defaults
  */
 export const fillInDefaults = (
   defaultValues: Record<string, any>,
   connectionSchema: {
-    properties: ConnectionTypeSecretSchemaReponse["properties"];
+    properties: ConnectionTypeSecretSchemaResponse["properties"];
   }
 ) => {
   const filledInValues = { ...defaultValues };