Alex/mvp UI for dbt cloud integration (#18095)

* Add lightly-styled ui for dbt cloud settings

* Add CollapsablePanel component

* Add CollapsablePanel around url input, MVP styling

To get the styling to work, I needed to edit `LabeledInput` to accept
a `className` prop, so I could give it contextually-specific styling.

* Add new feature flag for dbt cloud integration

This feature isn't added to either OSS or cloud builds; it will be
dynamically toggled for specific targeted accounts via LaunchDarkly's
`featureService.overwrites` key.

* Put settings page dbt cloud ui behind feature flag

* Add feature-flagged CloudTransformationsCard

* Extract (and rename) DbtCloudTransformationsCard

* Extract EmptyTransformationList component

* List transformations if any, "no integration" UI

This still uses some hardcoded conditions instead of anything resembling
actual data

* Initial UI for cloud transform jobs

* Use formik-backed inputs for job list data fields

* Improve job list management with FieldArray et al

* WIP: build payload to save job data as operations

There's some key data missing and it's not currently wired up

* Start pulling dbt cloud business logic to its own module

* Renaming pass (s/transformation/job/g)

* Move more logic into dbt service module

* Renaming pass (s/project/account/)

* Improve useDbtIntegration hook

* Add skeleton of updateWorkspace fn

* Connect pages to actual backend (no new jobs tho)

* Add hacky initial add new job implementation

* Put the whole dbt cloud card inside FieldArray

This dramatically simplifies adding to the list of jobs.

* Fix button placement, loss of focus on input

Never use the input prop in your component key, kids.

* re-extract DbtJobsList component

* Add input labels for dbt cloud job list

* Validate dbt cloud jobs so bad data doesn't crash the party

* Fix typo

* Improve dirty form tracking for dbt jobs list

* Remove unused input, add loading state to dbt cloud settings view

* Handle no integration, dirty states in dbt jobs list

Co-authored-by: Alex Birdsall <ambirdsall@gmail.com>

airbyte-webapp/src/components/LabeledInput/LabeledInput.tsx

@@ -3,21 +3,24 @@ import React from "react";
 import { ControlLabels, ControlLabelsProps } from "components/LabeledControl";
 import { Input, InputProps } from "components/ui/Input";
 
-type LabeledInputProps = Pick<ControlLabelsProps, "success" | "message" | "label" | "labelAdditionLength"> & InputProps;
+type LabeledInputProps = Pick<ControlLabelsProps, "success" | "message" | "label" | "labelAdditionLength"> &
+  InputProps & { className?: string };
 
 const LabeledInput: React.FC<LabeledInputProps> = ({
   error,
   success,
   message,
   label,
   labelAdditionLength,
+  className,
   ...inputProps
 }) => (
   <ControlLabels
     error={error}
     success={success}
     message={message}
     label={label}
+    className={className}
     labelAdditionLength={labelAdditionLength}
   >
     <Input {...inputProps} error={error} />

airbyte-webapp/src/hooks/services/Feature/types.tsx

@@ -1,6 +1,7 @@
 export enum FeatureItem {
   AllowUploadCustomImage = "ALLOW_UPLOAD_CUSTOM_IMAGE",
   AllowCustomDBT = "ALLOW_CUSTOM_DBT",
+  AllowDBTCloudIntegration = "ALLOW_DBT_CLOUD_INTEGRATION",
   AllowUpdateConnectors = "ALLOW_UPDATE_CONNECTORS",
   AllowOAuthConnector = "ALLOW_OAUTH_CONNECTOR",
   AllowCreateConnection = "ALLOW_CREATE_CONNECTION",

airbyte-webapp/src/packages/cloud/locales/en.json

@@ -86,6 +86,13 @@
   "settings.accountSettings.updateNameSuccess": "Your name has been updated!",
   "settings.userSettings": "User settings",
   "settings.workspaceSettings": "Workspace settings",
+  "settings.integrationSettings": "Integration settings",
+  "settings.integrationSettings.dbtCloudSettings": "dbt Cloud Integration",
+  "settings.integrationSettings.dbtCloudSettings.form.serviceToken": "Service Token",
+  "settings.integrationSettings.dbtCloudSettings.form.advancedOptions": "Advanced options",
+  "settings.integrationSettings.dbtCloudSettings.form.singleTenantUrl": "Single-tenant URL",
+  "settings.integrationSettings.dbtCloudSettings.form.testConnection": "Test connection",
+  "settings.integrationSettings.dbtCloudSettings.form.submit": "Save changes",
   "settings.generalSettings": "General Settings",
   "settings.generalSettings.changeWorkspace": "Change Workspace",
   "settings.generalSettings.form.name.label": "Workspace name",

airbyte-webapp/src/packages/cloud/services/dbtCloud.ts

@@ -0,0 +1,113 @@
+// This module is for the business logic of working with dbt Cloud webhooks.
+// Static config data, urls, functions which wrangle the APIs to manipulate
+// records in ways suited to the UI user workflows--all the implementation
+// details of working with dbtCloud jobs as webhook operations, all goes here.
+// The presentation logic and orchestration in the UI all goes elsewhere.
+//
+// About that business logic:
+// - for now, the code treats "webhook operations" and "dbt Cloud job" as synonymous.
+// - custom domains aren't yet supported
+
+import isEmpty from "lodash/isEmpty";
+import { useMutation } from "react-query";
+
+import { OperatorType, WebBackendConnectionRead, OperationRead } from "core/request/AirbyteClient";
+import { useWebConnectionService } from "hooks/services/useConnectionHook";
+import { useCurrentWorkspace } from "hooks/services/useWorkspace";
+import { useUpdateWorkspace } from "services/workspaces/WorkspacesService";
+
+export interface DbtCloudJob {
+  account: string;
+  job: string;
+  operationId?: string;
+}
+const dbtCloudDomain = "https://cloud.getdbt.com";
+const webhookConfigName = "dbt cloud";
+const executionBody = `{"cause": "airbyte"}`;
+const jobName = (t: DbtCloudJob) => `${t.account}/${t.job}`;
+
+const toDbtCloudJob = (operation: OperationRead): DbtCloudJob => {
+  const { operationId } = operation;
+  const { executionUrl } = operation.operatorConfiguration.webhook || {};
+
+  const matches = (executionUrl || "").match(/\/accounts\/([^/]+)\/jobs\/([^]+)\//);
+
+  if (!matches) {
+    throw new Error(`Cannot extract dbt cloud job params from executionUrl ${executionUrl}`);
+  } else {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const [_fullUrl, account, job] = matches;
+
+    return {
+      account,
+      job,
+      operationId,
+    };
+  }
+};
+const isDbtCloudJob = (operation: OperationRead): boolean =>
+  operation.operatorConfiguration.operatorType === OperatorType.webhook;
+
+export const useSubmitDbtCloudIntegrationConfig = () => {
+  const { workspaceId } = useCurrentWorkspace();
+  const { mutateAsync: updateWorkspace } = useUpdateWorkspace();
+
+  return useMutation(async (authToken: string) => {
+    await updateWorkspace({
+      workspaceId,
+      webhookConfigs: [
+        {
+          name: webhookConfigName,
+          authToken,
+        },
+      ],
+    });
+  });
+};
+
+export const useDbtIntegration = (connection: WebBackendConnectionRead) => {
+  const workspace = useCurrentWorkspace();
+  const { workspaceId } = workspace;
+  const connectionService = useWebConnectionService();
+
+  // TODO extract shared isDbtWebhookConfig predicate
+  const hasDbtIntegration = !isEmpty(workspace.webhookConfigs?.filter((config) => /dbt/.test(config.name || "")));
+  const webhookConfigId = workspace.webhookConfigs?.find((config) => /dbt/.test(config.name || ""))?.id;
+
+  const dbtCloudJobs = [...(connection.operations?.filter((operation) => isDbtCloudJob(operation)) || [])].map(
+    toDbtCloudJob
+  );
+  const otherOperations = [...(connection.operations?.filter((operation) => !isDbtCloudJob(operation)) || [])];
+
+  const saveJobs = (jobs: DbtCloudJob[]) => {
+    // TODO dynamically use the workspace's configured dbt cloud domain when it gets returned by backend
+    const urlForJob = (job: DbtCloudJob) => `${dbtCloudDomain}/api/v2/accounts/${job.account}/jobs/${job.job}/run`;
+
+    return connectionService.update({
+      connectionId: connection.connectionId,
+      operations: [
+        ...otherOperations,
+        ...jobs.map((job) => ({
+          workspaceId,
+          ...(job.operationId ? { operationId: job.operationId } : {}),
+          name: jobName(job),
+          operatorConfiguration: {
+            operatorType: OperatorType.webhook,
+            webhook: {
+              executionUrl: urlForJob(job),
+              // if `hasDbtIntegration` is true, webhookConfigId is guaranteed to exist
+              ...(webhookConfigId ? { webhookConfigId } : {}),
+              executionBody,
+            },
+          },
+        })),
+      ],
+    });
+  };
+
+  return {
+    hasDbtIntegration,
+    dbtCloudJobs,
+    saveJobs,
+  };
+};

airbyte-webapp/src/packages/cloud/views/settings/CloudSettingsPage.tsx

@@ -1,7 +1,9 @@
 import React, { useMemo } from "react";
 import { FormattedMessage } from "react-intl";
 
+import { FeatureItem, useFeature } from "hooks/services/Feature";
 // import useConnector from "hooks/services/useConnector";
+import { DbtCloudSettingsView } from "packages/cloud/views/settings/integrations/DbtCloudSettingsView";
 import { AccountSettingsView } from "packages/cloud/views/users/AccountSettingsView";
 import { UsersSettingsView } from "packages/cloud/views/users/UsersSettingsView";
 import { WorkspaceSettingsView } from "packages/cloud/views/workspaces/WorkspaceSettingsView";
@@ -20,6 +22,7 @@ import { CloudSettingsRoutes } from "./routePaths";
 export const CloudSettingsPage: React.FC = () => {
   // TODO: uncomment when supported in cloud
   // const { countNewSourceVersion, countNewDestinationVersion } = useConnector();
+  const supportsCloudDbtIntegration = useFeature(FeatureItem.AllowDBTCloudIntegration);
 
   const pageConfig = useMemo<PageConfig>(
     () => ({
@@ -82,9 +85,24 @@ export const CloudSettingsPage: React.FC = () => {
             },
           ],
         },
+        ...(supportsCloudDbtIntegration
+          ? [
+              {
+                category: <FormattedMessage id="settings.integrationSettings" />,
+                routes: [
+                  {
+                    path: CloudSettingsRoutes.DbtCloud,
+                    name: <FormattedMessage id="settings.integrationSettings.dbtCloudSettings" />,
+                    component: DbtCloudSettingsView,
+                    id: "integrationSettings.dbtCloudSettings",
+                  },
+                ],
+              },
+            ]
+          : []),
       ],
     }),
-    []
+    [supportsCloudDbtIntegration]
   );
 
   return <SettingsPage pageConfig={pageConfig} />;

airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.module.scss

@@ -0,0 +1,14 @@
+@use "scss/colors";
+@use "scss/variables" as vars;
+
+$item-spacing: 25px;
+
+.controlGroup {
+  display: flex;
+  justify-content: flex-end;
+  margin-top: $item-spacing;
+
+  .button {
+    margin-left: 1em;
+  }
+}

airbyte-webapp/src/packages/cloud/views/settings/integrations/DbtCloudSettingsView.tsx

@@ -0,0 +1,44 @@
+import { Field, FieldProps, Form, Formik } from "formik";
+import React from "react";
+import { FormattedMessage } from "react-intl";
+
+import { LabeledInput } from "components/LabeledInput";
+import { Button } from "components/ui/Button";
+
+import { useSubmitDbtCloudIntegrationConfig } from "packages/cloud/services/dbtCloud";
+import { Content, SettingsCard } from "pages/SettingsPage/pages/SettingsComponents";
+
+import styles from "./DbtCloudSettingsView.module.scss";
+
+export const DbtCloudSettingsView: React.FC = () => {
+  const { mutate: submitDbtCloudIntegrationConfig, isLoading } = useSubmitDbtCloudIntegrationConfig();
+  return (
+    <SettingsCard title={<FormattedMessage id="settings.integrationSettings.dbtCloudSettings" />}>
+      <Content>
+        <Formik
+          initialValues={{
+            serviceToken: "",
+          }}
+          onSubmit={({ serviceToken }) => submitDbtCloudIntegrationConfig(serviceToken)}
+        >
+          <Form>
+            <Field name="serviceToken">
+              {({ field }: FieldProps<string>) => (
+                <LabeledInput
+                  {...field}
+                  label={<FormattedMessage id="settings.integrationSettings.dbtCloudSettings.form.serviceToken" />}
+                  type="text"
+                />
+              )}
+            </Field>
+            <div className={styles.controlGroup}>
+              <Button variant="primary" type="submit" className={styles.button} isLoading={isLoading}>
+                <FormattedMessage id="settings.integrationSettings.dbtCloudSettings.form.submit" />
+              </Button>
+            </div>
+          </Form>
+        </Formik>
+      </Content>
+    </SettingsCard>
+  );
+};

airbyte-webapp/src/packages/cloud/views/settings/routePaths.ts

@@ -9,4 +9,5 @@ export const CloudSettingsRoutes = {
 
   Workspace: "workspaces",
   AccessManagement: "access-management",
+  DbtCloud: "dbt-cloud",
 } as const;

airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx

@@ -25,6 +25,7 @@ import {
 import { FormCard } from "views/Connection/FormCard";
 
 import styles from "./ConnectionTransformationTab.module.scss";
+import { DbtCloudTransformationsCard } from "./ConnectionTransformationTab/DbtCloudTransformationsCard";
 
 const CustomTransformationsCard: React.FC<{
   operations?: OperationCreate[];
@@ -100,6 +101,8 @@ export const ConnectionTransformationTab: React.FC = () => {
   useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_TRANSFORMATION);
   const { supportsNormalization } = definition;
   const supportsDbt = useFeature(FeatureItem.AllowCustomDBT) && definition.supportsDbt;
+  const supportsCloudDbtIntegration = useFeature(FeatureItem.AllowDBTCloudIntegration) && definition.supportsDbt;
+  const noSupportedTransformations = !supportsNormalization && !supportsDbt && !supportsCloudDbtIntegration;
 
   const onSubmit: FormikOnSubmit<{ transformations?: OperationRead[]; normalization?: NormalizationType }> = async (
     values,
@@ -134,7 +137,8 @@ export const ConnectionTransformationTab: React.FC = () => {
       >
         {supportsNormalization && <NormalizationCard operations={connection.operations} onSubmit={onSubmit} />}
         {supportsDbt && <CustomTransformationsCard operations={connection.operations} onSubmit={onSubmit} />}
-        {!supportsNormalization && !supportsDbt && (
+        {supportsCloudDbtIntegration && <DbtCloudTransformationsCard connection={connection} />}
+        {noSupportedTransformations && (
           <Card className={styles.customCard}>
             <Text as="p" size="lg" centered>
               <FormattedMessage id="connectionForm.operations.notSupported" />

airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab/DbtCloudTransformationsCard.module.scss

@@ -0,0 +1,91 @@
+@use "scss/colors";
+
+.jobListContainer {
+  padding: 25px 25px 22px;
+  background-color: colors.$grey-50;
+}
+
+.jobListTitle {
+  display: flex;
+  justify-content: space-between;
+}
+
+.jobListForm {
+  width: 100%;
+}
+
+.emptyListContent {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+
+  > img {
+    width: 111px;
+    height: 111px;
+    margin: 20px 0;
+  }
+}
+
+.contextExplanation {
+  color: colors.$grey-300;
+  width: 100%;
+
+  & a {
+    color: colors.$grey-300;
+  }
+}
+
+.jobListButtonGroup {
+  display: flex;
+  justify-content: flex-end;
+  margin-top: 20px;
+  width: 100%;
+}
+
+.jobListButton {
+  margin-left: 10px;
+}
+
+.jobListItem {
+  margin-top: 10px;
+  padding: 18px;
+  width: 100%;
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+
+  & img {
+    height: 32px;
+    width: 32px;
+  }
+}
+
+.jobListItemIntegrationName {
+  display: flex;
+  align-items: center;
+}
+
+.jobListItemInputGroup {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+
+.jobListItemInput {
+  height: fit-content;
+  margin-left: 1em;
+}
+
+.jobListItemInputLabel {
+  font-size: 11px;
+  font-weight: 500;
+}
+
+.jobListItemDelete {
+  color: colors.$grey-200;
+  font-size: large;
+  margin: 0 1em;
+  cursor: pointer;
+  border: none;
+  background-color: inherit;
+}