From 699470e26dff35b503d3a42fabf1b5810dff6b44 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 1 Dec 2022 15:01:55 -0800 Subject: [PATCH 001/100] move connector builder components into the same shared components/connectorBuilder directory --- .../StreamTestingPanel/ConfigMenu.module.scss | 0 .../{ => connectorBuilder}/StreamTestingPanel/ConfigMenu.tsx | 0 .../StreamTestingPanel/LogsDisplay.module.scss | 0 .../{ => connectorBuilder}/StreamTestingPanel/LogsDisplay.tsx | 0 .../StreamTestingPanel/PageDisplay.module.scss | 0 .../{ => connectorBuilder}/StreamTestingPanel/PageDisplay.tsx | 0 .../StreamTestingPanel/ResultDisplay.module.scss | 0 .../StreamTestingPanel/ResultDisplay.tsx | 0 .../StreamTestingPanel/SliceSelector.module.scss | 0 .../StreamTestingPanel/SliceSelector.tsx | 0 .../StreamTestingPanel/StreamSelector.module.scss | 0 .../StreamTestingPanel/StreamSelector.tsx | 0 .../StreamTestingPanel/StreamTester.module.scss | 0 .../StreamTestingPanel/StreamTester.tsx | 0 .../StreamTestingPanel/StreamTestingPanel.module.scss | 0 .../StreamTestingPanel/StreamTestingPanel.tsx | 0 .../{ => connectorBuilder}/StreamTestingPanel/index.tsx | 0 .../{ => connectorBuilder}/StreamTestingPanel/utils.ts | 0 .../{ => connectorBuilder}/YamlEditor/DownloadYamlButton.tsx | 0 .../{ => connectorBuilder}/YamlEditor/YamlEditor.module.scss | 0 .../{ => connectorBuilder}/YamlEditor/YamlEditor.tsx | 0 .../components/{ => connectorBuilder}/YamlEditor/index.tsx | 0 .../src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx | 4 ++-- 23 files changed, 2 insertions(+), 2 deletions(-) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/ConfigMenu.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/ConfigMenu.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/LogsDisplay.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/LogsDisplay.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/PageDisplay.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/PageDisplay.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/ResultDisplay.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/ResultDisplay.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/SliceSelector.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/SliceSelector.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamSelector.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamSelector.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamTester.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamTester.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamTestingPanel.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/StreamTestingPanel.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/index.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/StreamTestingPanel/utils.ts (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/YamlEditor/DownloadYamlButton.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/YamlEditor/YamlEditor.module.scss (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/YamlEditor/YamlEditor.tsx (100%) rename airbyte-webapp/src/components/{ => connectorBuilder}/YamlEditor/index.tsx (100%) diff --git a/airbyte-webapp/src/components/StreamTestingPanel/ConfigMenu.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/ConfigMenu.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/ConfigMenu.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/LogsDisplay.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/LogsDisplay.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/LogsDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/LogsDisplay.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/LogsDisplay.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/PageDisplay.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/PageDisplay.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/PageDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/PageDisplay.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/ResultDisplay.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ResultDisplay.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/ResultDisplay.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ResultDisplay.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/ResultDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ResultDisplay.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/ResultDisplay.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ResultDisplay.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/SliceSelector.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/SliceSelector.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/SliceSelector.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/SliceSelector.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/SliceSelector.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamSelector.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamSelector.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamSelector.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamSelector.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamTester.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamTester.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamTester.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamTestingPanel.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamTestingPanel.module.scss rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss diff --git a/airbyte-webapp/src/components/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/StreamTestingPanel.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/index.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/index.tsx similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/index.tsx rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/index.tsx diff --git a/airbyte-webapp/src/components/StreamTestingPanel/utils.ts b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/utils.ts similarity index 100% rename from airbyte-webapp/src/components/StreamTestingPanel/utils.ts rename to airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/utils.ts diff --git a/airbyte-webapp/src/components/YamlEditor/DownloadYamlButton.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx similarity index 100% rename from airbyte-webapp/src/components/YamlEditor/DownloadYamlButton.tsx rename to airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx diff --git a/airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss similarity index 100% rename from airbyte-webapp/src/components/YamlEditor/YamlEditor.module.scss rename to airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss diff --git a/airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx similarity index 100% rename from airbyte-webapp/src/components/YamlEditor/YamlEditor.tsx rename to airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx diff --git a/airbyte-webapp/src/components/YamlEditor/index.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/index.tsx similarity index 100% rename from airbyte-webapp/src/components/YamlEditor/index.tsx rename to airbyte-webapp/src/components/connectorBuilder/YamlEditor/index.tsx diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx index e4d3cb1bd8f0..54e5de58cd68 100644 --- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx +++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx @@ -1,8 +1,8 @@ import { useIntl } from "react-intl"; -import { StreamTestingPanel } from "components/StreamTestingPanel"; +import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel"; +import { YamlEditor } from "components/connectorBuilder/YamlEditor"; import { ResizablePanels } from "components/ui/ResizablePanels"; -import { YamlEditor } from "components/YamlEditor"; import { ConnectorBuilderStateProvider } from "services/connectorBuilder/ConnectorBuilderStateService"; From 827532bbb374dd7635c4de07d28967af36178a71 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 1 Dec 2022 15:21:02 -0800 Subject: [PATCH 002/100] move diff over from poc branch --- airbyte-webapp/orval.config.ts | 17 +++ .../Builder/Builder.module.scss | 21 ++++ .../connectorBuilder/Builder/Builder.tsx | 90 +++++++++++++++ .../Builder/BuilderCard.module.scss | 8 ++ .../connectorBuilder/Builder/BuilderCard.tsx | 9 ++ .../Builder/BuilderField.module.scss | 13 +++ .../connectorBuilder/Builder/BuilderField.tsx | 107 ++++++++++++++++++ .../Builder/BuilderSidebar.module.scss | 35 ++++++ .../Builder/BuilderSidebar.tsx | 29 +++++ .../Builder/UiYamlToggleButton.module.scss | 33 ++++++ .../Builder/UiYamlToggleButton.tsx | 38 +++++++ .../StreamSelector.module.scss | 2 +- .../StreamTestingPanel.module.scss | 2 +- .../YamlEditor/DownloadYamlButton.module.scss | 4 + .../YamlEditor/DownloadYamlButton.tsx | 20 ++-- .../YamlEditor/YamlEditor.module.scss | 15 ++- .../YamlEditor/YamlEditor.tsx | 28 +++-- .../ConnectorBuilderPage.module.scss | 10 +- .../ConnectorBuilderPage.tsx | 16 ++- airbyte-webapp/src/pages/routes.tsx | 2 +- .../ConnectorBuilderStateService.tsx | 45 ++++++-- 21 files changed, 504 insertions(+), 40 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.module.scss diff --git a/airbyte-webapp/orval.config.ts b/airbyte-webapp/orval.config.ts index f859eaa5e4e2..bd41b407de99 100644 --- a/airbyte-webapp/orval.config.ts +++ b/airbyte-webapp/orval.config.ts @@ -43,4 +43,21 @@ export default defineConfig({ }, }, }, + connectorManifest: { + input: "../airbyte-cdk/python/airbyte_cdk/sources/declarative/connector_manifest_schema_openapi.json", + output: { + target: "./src/core/request/ConnectorManifest.ts", + prettier: true, + override: { + header: (info) => [ + `eslint-disable`, + `Generated by orval 🍺`, + `Do not edit manually. Run "npm run generate-client" instead.`, + ...(info.title ? [info.title] : []), + ...(info.description ? [info.description] : []), + ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []), + ], + }, + }, + }, }); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss new file mode 100644 index 000000000000..eeb67b740c42 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss @@ -0,0 +1,21 @@ +@use "scss/variables"; +@use "scss/colors"; + +.container { + height: 100%; + display: flex; +} + +.sidebar { + flex: 0 0 200px; + background-color: colors.$white; + box-shadow: 2px 0 10px rgba(26, 25, 77, 10%); +} + +.form { + flex: 1; + padding: variables.$spacing-xl; + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx new file mode 100644 index 000000000000..cfb44c3e064b --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -0,0 +1,90 @@ +import { Form, Formik, useFormikContext } from "formik"; +import { useEffect } from "react"; + +import { ConnectorManifest } from "core/request/ConnectorManifest"; +import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { usePatchFormik } from "views/Connector/ConnectorForm/useBuildForm"; + +import styles from "./Builder.module.scss"; +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; +import { BuilderSidebar } from "./BuilderSidebar"; + +const FormikPatch: React.FC = () => { + usePatchFormik(); + return null; +}; + +const FormObserver: React.FC = () => { + const { values } = useFormikContext(); + const { setJsonManifest } = useConnectorBuilderState(); + + useEffect(() => { + setJsonManifest(values); + }, [values, setJsonManifest]); + + return null; +}; + +interface BuilderProps { + toggleYamlEditor: () => void; +} + +export const Builder: React.FC = ({ toggleYamlEditor }) => { + const { jsonManifest } = useConnectorBuilderState(); + + return ( + { + console.log(values); + }} + > + <> + +
+ +
+ + + {/* Note: we are explicitly NOT using intl for the BuilderField strings, in order to keep this easier to maintain */} + + + + + + + + + +
+ + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss new file mode 100644 index 000000000000..48fedff9a2a0 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss @@ -0,0 +1,8 @@ +@use "scss/variables"; + +.card { + padding: variables.$spacing-xl; + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx new file mode 100644 index 000000000000..9bbb968a3d29 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx @@ -0,0 +1,9 @@ +import React from "react"; + +import { Card } from "components/ui/Card"; + +import styles from "./BuilderCard.module.scss"; + +export const BuilderCard: React.FC> = ({ children }) => { + return {children}; +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss new file mode 100644 index 000000000000..332592b3a616 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss @@ -0,0 +1,13 @@ +@use "scss/variables"; +@use "scss/colors"; + +// .container { +// display: flex; +// flex-direction: column; +// gap: 0; +// } + +.error { + margin-top: variables.$spacing-sm; + color: colors.$red; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx new file mode 100644 index 000000000000..029ca95c2353 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -0,0 +1,107 @@ +import { useField } from "formik"; +import { useEffect } from "react"; +import { FormattedMessage } from "react-intl"; +import * as yup from "yup"; + +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; +import { TagInput } from "components/ui/TagInput"; +import { Text } from "components/ui/Text"; + +import styles from "./BuilderField.module.scss"; + +interface EnumFieldProps { + options: string[]; + value: string; + setValue: (value: string) => void; + error: boolean; +} + +interface ArrayFieldProps { + name: string; + value: string[]; + setValue: (value: string[]) => void; + error: boolean; +} + +interface BaseFieldProps { + // path to the location in the Connector Manifest schema which should be set by this component + path: string; + label: string; + tooltip?: string; + optional?: boolean; +} + +type BuilderFieldProps = BaseFieldProps & ({ type: "text" } | { type: "array" } | { type: "enum"; options: string[] }); + +const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { + useEffect(() => { + setValue(value); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + return ( + { + return { label: option, value: option }; + })} + onChange={(selected) => selected && setValue(selected.value)} + value={value} + error={error} + /> + ); +}; + +const ArrayField: React.FC = ({ name, value, setValue, error }) => { + useEffect(() => { + setValue(value); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + return setValue(value)} error={error} />; +}; + +export const BuilderField: React.FC = ({ path, label, tooltip, optional = false, ...props }) => { + let yupSchema = props.type === "array" ? yup.array().of(yup.string()) : yup.string(); + if (!optional) { + yupSchema = yupSchema.required("form.empty.error"); + } + const fieldConfig = { + name: path, + validate: (value: string) => { + try { + yupSchema.validateSync(value); + return undefined; + } catch (err) { + if (err instanceof yup.ValidationError) { + return err.errors.join(", "); + } + throw err; + } + }, + }; + const [field, meta, helpers] = useField(fieldConfig); + const hasError = !!meta.error && meta.touched; + + return ( + + {props.type === "text" && } + {props.type === "array" && ( + + )} + {props.type === "enum" && ( + + )} + {hasError && ( + + + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss new file mode 100644 index 000000000000..3fad3fbfabfb --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -0,0 +1,35 @@ +@use "scss/variables"; +@use "scss/colors"; + +.container { + // padding: variables.$spacing-sm; + display: flex; + flex-direction: column; + align-items: center; +} + +.yamlToggle { + margin-top: variables.$spacing-lg; +} + +.connectorImg { + width: 90px; + box-shadow: 0 2px 4px rgba(26, 25, 77, 12%); + border-radius: 25px; + padding: variables.$spacing-xl; + margin-top: variables.$spacing-xl; +} + +.connectorName { + margin-top: variables.$spacing-md; + height: 28px; + display: flex; + align-items: center; + justify-content: center; +} + +.downloadButton { + margin-top: auto; + margin-bottom: variables.$spacing-md; + width: 85%; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx new file mode 100644 index 000000000000..ff8e777ed8a6 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -0,0 +1,29 @@ +import classnames from "classnames"; + +import { Heading } from "components/ui/Heading"; + +import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; + +import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; +import styles from "./BuilderSidebar.module.scss"; +import { UiYamlToggleButton } from "./UiYamlToggleButton"; + +interface BuilderSidebarProps { + className?: string; + toggleYamlEditor: () => void; +} + +export const BuilderSidebar: React.FC = ({ className, toggleYamlEditor }) => { + const { yamlManifest } = useConnectorBuilderState(); + + return ( +
+ + Connector Logo + + Connector Name + + +
+ ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss new file mode 100644 index 000000000000..c356d88cef9b --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss @@ -0,0 +1,33 @@ +@use "scss/variables"; +@use "scss/colors"; + +.button { + cursor: pointer; + border: variables.$border-thin solid colors.$dark-blue; + background-color: colors.$dark-blue; + display: flex; + border-radius: 6px; + padding: 0; + overflow: hidden; + height: 21px; + width: 96px; +} + +.text { + height: 100%; + flex: 1; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; +} + +.selected { + background-color: colors.$white; + color: colors.$dark-blue; +} + +.unselected { + background-color: transparent; + color: colors.$white; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx new file mode 100644 index 000000000000..9e970847eb3b --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx @@ -0,0 +1,38 @@ +import classnames from "classnames"; + +import { Text } from "components/ui/Text"; + +import styles from "./UiYamlToggleButton.module.scss"; + +interface UiYamlToggleButtonProps { + className?: string; + yamlSelected: boolean; + onClick: () => void; +} + +export const UiYamlToggleButton: React.FC = ({ className, yamlSelected, onClick }) => { + return ( + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss index 7f297b2eb5cf..4768d440fe12 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss @@ -11,7 +11,7 @@ .button { padding: variables.$spacing-md; background-color: transparent; - border-radius: variables.$border-radius-lg; + border-radius: variables.$border-radius-md; border: none; display: flex; justify-content: center; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss index 43a610209f84..30a5b8480227 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss @@ -38,7 +38,7 @@ $buttonHeight: 36px; flex-direction: column; gap: variables.$spacing-md; background-color: colors.$blue-50; - border-radius: variables.$border-radius-md; + border-radius: variables.$border-radius-sm; } .loadingSpinner { diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.module.scss new file mode 100644 index 000000000000..5d63d46a8428 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.module.scss @@ -0,0 +1,4 @@ +.button { + width: 100%; + height: 100%; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx index f5a49a5fc99e..1efbd049346a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx @@ -7,6 +7,8 @@ import { Tooltip } from "components/ui/Tooltip"; import { downloadFile } from "utils/file"; +import styles from "./DownloadYamlButton.module.scss"; + interface DownloadYamlButtonProps { className?: string; yaml: string; @@ -22,7 +24,7 @@ export const DownloadYamlButton: React.FC = ({ classNam const downloadButton = ( ); - return yamlIsValid ? ( - downloadButton - ) : ( - - - + return ( +
+ {yamlIsValid ? ( + downloadButton + ) : ( + + + + )} +
); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss index 6d7b4b3d1890..ecc27ad66cfc 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss @@ -9,13 +9,24 @@ .control { flex: 0 0 auto; - background-color: transparent; + background-color: colors.$grey-30; display: flex; padding: variables.$spacing-md; flex-direction: row; - justify-content: flex-end; + + // justify-content: flex-end; } .editorContainer { + padding-top: variables.$spacing-xl; flex: 1 1 0; } + +.yamlToggle { + margin-left: 42px; + margin-top: 5px; +} + +.downloadButton { + margin-left: auto; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 91df9c5330c6..3bb5a311e36f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -2,25 +2,28 @@ import { useMonaco } from "@monaco-editor/react"; import { load, YAMLException } from "js-yaml"; import { editor } from "monaco-editor/esm/vs/editor/editor.api"; import { useEffect, useRef, useState } from "react"; -import { useDebounce, useLocalStorage } from "react-use"; import { CodeEditor } from "components/ui/CodeEditor"; -import { StreamsListRequestBodyManifest } from "core/request/ConnectorBuilderClient"; -import { useManifestTemplate } from "services/connectorBuilder/ConnectorBuilderApiService"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { UiYamlToggleButton } from "../Builder/UiYamlToggleButton"; import { DownloadYamlButton } from "./DownloadYamlButton"; import styles from "./YamlEditor.module.scss"; -export const YamlEditor: React.FC = () => { - const yamlEditorRef = useRef(); - const template = useManifestTemplate(); - const [locallyStoredYaml, setLocallyStoredYaml] = useLocalStorage("connectorBuilderYaml", template); - const [yamlValue, setYamlValue] = useState(locallyStoredYaml ?? template); - useDebounce(() => setLocallyStoredYaml(yamlValue), 500, [yamlValue]); +interface YamlEditorProps { + toggleYamlEditor: () => void; +} - const { yamlIsValid, setYamlEditorIsMounted, setYamlIsValid, setJsonManifest } = useConnectorBuilderState(); +export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { + const yamlEditorRef = useRef(); + // const template = useManifestTemplate(); + // const [locallyStoredYaml, setLocallyStoredYaml] = useLocalStorage("connectorBuilderYaml", template); + // useDebounce(() => setLocallyStoredYaml(yamlValue), 500, [yamlValue]); + const { yamlManifest, yamlIsValid, setYamlEditorIsMounted, setYamlIsValid, setJsonManifest } = + useConnectorBuilderState(); + const [yamlValue, setYamlValue] = useState(yamlManifest); const monaco = useMonaco(); @@ -30,7 +33,7 @@ export const YamlEditor: React.FC = () => { const yamlEditorModel = yamlEditorRef.current.getModel(); try { - const json = load(yamlValue) as StreamsListRequestBodyManifest; + const json = load(yamlValue) as ConnectorManifest; setJsonManifest(json); setYamlIsValid(true); @@ -64,7 +67,8 @@ export const YamlEditor: React.FC = () => { return (
- + +
{ const { formatMessage } = useIntl(); + const [showYamlEditor, toggleYamlEditor] = useToggle(true); return ( , + children: ( + <> + {showYamlEditor ? ( + + ) : ( + + )} + + ), className: styles.leftPanel, minWidth: 100, }} diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 6d7edd57d25c..2d758484a771 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -55,7 +55,6 @@ const MainViewRoutes: React.FC = () => { } /> } /> } /> - } /> } /> @@ -109,6 +108,7 @@ export const Routing: React.FC = () => { ); return ( + } /> {OldRoutes} } /> } /> diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index e5e9bcbeb659..7d61aa766202 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -1,17 +1,17 @@ +import { dump } from "js-yaml"; import React, { useContext, useEffect, useMemo, useState } from "react"; import { useIntl } from "react-intl"; +import { useLocalStorage } from "react-use"; -import { - StreamReadRequestBodyConfig, - StreamsListReadStreamsItem, - StreamsListRequestBodyManifest, -} from "core/request/ConnectorBuilderClient"; +import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useListStreams } from "./ConnectorBuilderApiService"; interface Context { - jsonManifest: StreamsListRequestBodyManifest; + jsonManifest: ConnectorManifest; + yamlManifest: string; yamlEditorIsMounted: boolean; yamlIsValid: boolean; streams: StreamsListReadStreamsItem[]; @@ -19,7 +19,8 @@ interface Context { selectedStream?: StreamsListReadStreamsItem; configString: string; configJson: StreamReadRequestBodyConfig; - setJsonManifest: (jsonValue: StreamsListRequestBodyManifest) => void; + setJsonManifest: (jsonValue: ConnectorManifest) => void; + setYamlManifest: (yamlValue: string) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setSelectedStream: (streamName: string) => void; @@ -32,9 +33,25 @@ export const ConnectorBuilderStateProvider: React.FC({}); + const defaultJsonManifest = { + version: "1.0.0", + check: { + stream_names: [], + }, + streams: [], + }; + const [jsonManifest, setJsonManifest] = useLocalStorage( + "connectorBuilderManifest", + defaultJsonManifest + ); + const manifest = jsonManifest ?? defaultJsonManifest; const [yamlIsValid, setYamlIsValid] = useState(true); - const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(false); + const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(true); + + const [yamlManifest, setYamlManifest] = useState(""); + useEffect(() => { + setYamlManifest(dump(jsonManifest)); + }, [jsonManifest]); // config const [configString, setConfigString] = useState("{\n \n}"); @@ -49,12 +66,14 @@ export const ConnectorBuilderStateProvider: React.FC { return streamListRead?.streams ?? []; }, [streamListRead]); + // const streamListErrorMessage = undefined; + // const streams = useMemo(() => [{ name: "stream1", url: "url1" }], []); const firstStreamName = streams.length > 0 ? streams[0].name : undefined; const [selectedStreamName, setSelectedStream] = useState(firstStreamName); @@ -78,7 +99,8 @@ export const ConnectorBuilderStateProvider: React.FC stream.name === selectedStreamName); const ctx = { - jsonManifest, + jsonManifest: manifest, + yamlManifest, yamlEditorIsMounted, yamlIsValid, streams, @@ -87,6 +109,7 @@ export const ConnectorBuilderStateProvider: React.FC Date: Fri, 2 Dec 2022 16:04:10 -0800 Subject: [PATCH 003/100] save current progress --- .../Builder/AddStreamButton.tsx | 50 +++++++++ .../connectorBuilder/Builder/Builder.tsx | 101 ++++++------------ .../connectorBuilder/Builder/BuilderField.tsx | 15 +-- .../Builder/BuilderSidebar.tsx | 49 ++++++++- .../Builder/GlobalConfigView.tsx | 15 +++ .../Builder/StreamConfigView.tsx | 35 ++++++ .../connectorBuilder/Builder/types.ts | 51 +++++++++ .../ConnectorBuilderPage.tsx | 72 ++++++++----- .../ConnectorBuilderStateService.tsx | 52 ++++++--- 9 files changed, 320 insertions(+), 120 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/types.ts diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx new file mode 100644 index 000000000000..8842b3b1713a --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -0,0 +1,50 @@ +import { Formik } from "formik"; +import { useState } from "react"; + +import { Button } from "components/ui/Button"; +import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; + +import { FormikPatch } from "./Builder"; +import { BuilderField } from "./BuilderField"; + +interface AddStreamValues { + streamName: string; + urlPath: string; +} + +export const AddStreamButton: React.FC = () => { + const [isOpen, setIsOpen] = useState(false); + + return ( + <> + + {isOpen && ( + setIsOpen(false)}> + { + console.log(values); + setIsOpen(false); + }} + > + <> + + + + + + + + + + + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index cfb44c3e064b..f2640ef42d32 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -1,90 +1,49 @@ -import { Form, Formik, useFormikContext } from "formik"; -import { useEffect } from "react"; +import { Form } from "formik"; +import { useEffect, useState } from "react"; -import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { usePatchFormik } from "views/Connector/ConnectorForm/useBuildForm"; import styles from "./Builder.module.scss"; -import { BuilderCard } from "./BuilderCard"; -import { BuilderField } from "./BuilderField"; import { BuilderSidebar } from "./BuilderSidebar"; +import { GlobalConfigView } from "./GlobalConfigView"; +import { StreamConfigView } from "./StreamConfigView"; +import { BuilderFormValues } from "./types"; -const FormikPatch: React.FC = () => { +export const FormikPatch: React.FC = () => { usePatchFormik(); return null; }; -const FormObserver: React.FC = () => { - const { values } = useFormikContext(); - const { setJsonManifest } = useConnectorBuilderState(); - - useEffect(() => { - setJsonManifest(values); - }, [values, setJsonManifest]); - - return null; -}; - interface BuilderProps { + values: BuilderFormValues; toggleYamlEditor: () => void; } -export const Builder: React.FC = ({ toggleYamlEditor }) => { - const { jsonManifest } = useConnectorBuilderState(); +export const Builder: React.FC = ({ values, toggleYamlEditor }) => { + const { setBuilderFormValues } = useConnectorBuilderState(); + useEffect(() => { + setBuilderFormValues(values); + }, [values, setBuilderFormValues]); + + const [selectedView, setSelectedView] = useState<"global" | number>("global"); + + console.log("values", values); return ( - { - console.log(values); - }} - > - <> - -
- -
- - - {/* Note: we are explicitly NOT using intl for the BuilderField strings, in order to keep this easier to maintain */} - - - - - - - - - -
- - + <> + +
+ +
+ {selectedView === "global" ? : } + +
+ ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 029ca95c2353..80eef9b28a3c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -1,5 +1,4 @@ import { useField } from "formik"; -import { useEffect } from "react"; import { FormattedMessage } from "react-intl"; import * as yup from "yup"; @@ -36,9 +35,9 @@ interface BaseFieldProps { type BuilderFieldProps = BaseFieldProps & ({ type: "text" } | { type: "array" } | { type: "enum"; options: string[] }); const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { - useEffect(() => { - setValue(value); - }, []); // eslint-disable-line react-hooks/exhaustive-deps + // useEffect(() => { + // setValue(value); + // }, []); // eslint-disable-line react-hooks/exhaustive-deps return ( = ({ options, value, setValue, error, }; const ArrayField: React.FC = ({ name, value, setValue, error }) => { - useEffect(() => { - setValue(value); - }, []); // eslint-disable-line react-hooks/exhaustive-deps + // useEffect(() => { + // setValue(value); + // }, []); // eslint-disable-line react-hooks/exhaustive-deps return setValue(value)} error={error} />; }; @@ -83,6 +82,8 @@ export const BuilderField: React.FC = ({ path, label, tooltip const [field, meta, helpers] = useField(fieldConfig); const hasError = !!meta.error && meta.touched; + console.log(`path: ${path}, value: ${field.value}`); + return ( {props.type === "text" && } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index ff8e777ed8a6..70a9ef4b8433 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,20 +1,50 @@ import classnames from "classnames"; +import { useField, useFormikContext } from "formik"; import { Heading } from "components/ui/Heading"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; +import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; import { UiYamlToggleButton } from "./UiYamlToggleButton"; +export type BuilderView = "global" | number; + +interface StreamSelectButtonProps { + streamPath: string; + onClick: () => void; +} + +const StreamSelectButton: React.FC = ({ streamPath, onClick }) => { + const streamNamePath = `${streamPath}.name`; + console.log("streamNamePath", streamNamePath); + const [field] = useField(streamNamePath); + console.log("field.value", field.value); + + return ; +}; + interface BuilderSidebarProps { className?: string; toggleYamlEditor: () => void; + numStreams: number; + onViewSelect: (selected: BuilderView) => void; } -export const BuilderSidebar: React.FC = ({ className, toggleYamlEditor }) => { - const { yamlManifest } = useConnectorBuilderState(); +export const BuilderSidebar: React.FC = ({ + className, + toggleYamlEditor, + numStreams, + onViewSelect, +}) => { + const { yamlManifest, resetBuilderFormValues } = useConnectorBuilderState(); + const { resetForm } = useFormikContext(); + const handleResetForm = () => { + resetBuilderFormValues(); + resetForm(); + }; return (
@@ -23,7 +53,22 @@ export const BuilderSidebar: React.FC = ({ className, toggl Connector Name + + + + + Streams + + + + + {Array.from(Array(numStreams).keys()).map((streamNum) => { + const streamPath = `streams[${streamNum}]`; + return onViewSelect(streamNum)} />; + })} + +
); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx new file mode 100644 index 000000000000..49c060dc7617 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -0,0 +1,15 @@ +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; + +export const GlobalConfigView: React.FC = () => { + return ( + <> + + + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx new file mode 100644 index 000000000000..6c56fcd03fc9 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -0,0 +1,35 @@ +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; + +interface StreamConfigViewProps { + streamNum: number; +} + +export const StreamConfigView: React.FC = ({ streamNum }) => { + const streamPath = (path: string) => `streams[${streamNum}].${path}`; + + return ( + + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts b/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts new file mode 100644 index 000000000000..ce3625b13fe7 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts @@ -0,0 +1,51 @@ +import { ConnectorManifest, DeclarativeStream, HttpRequesterAllOfAuthenticator } from "core/request/ConnectorManifest"; + +export interface BuilderFormValues { + connectorName: string; + urlBase: string; + // TODO: make required when authenticator is fully added + authenticator?: HttpRequesterAllOfAuthenticator; + streams: BuilderStream[]; +} + +export interface BuilderStream { + name: string; + urlPath: string; + fieldPointer: string[]; + httpMethod: "GET" | "POST"; +} + +export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { + const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { + return { + name: stream.name, + retriever: { + name: stream.name, + requester: { + name: stream.name, + url_base: values.urlBase, + path: stream.urlPath, + authenticator: values.authenticator, + // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema + config: {}, + }, + record_selector: { + extractor: { + field_pointer: stream.fieldPointer, + config: {}, + }, + }, + config: {}, + }, + config: {}, + }; + }); + + return { + version: "0.1.0", + check: { + stream_names: [], + }, + streams: manifestStreams, + }; +}; diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx index 85cf5d1157c2..9b7e3af62c1e 100644 --- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx +++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx @@ -1,48 +1,64 @@ import classnames from "classnames"; +import { Formik } from "formik"; import { useIntl } from "react-intl"; import { useToggle } from "react-use"; import { Builder } from "components/connectorBuilder/Builder/Builder"; +import { BuilderFormValues } from "components/connectorBuilder/Builder/types"; import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel"; import { YamlEditor } from "components/connectorBuilder/YamlEditor"; import { ResizablePanels } from "components/ui/ResizablePanels"; -import { ConnectorBuilderStateProvider } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { + ConnectorBuilderStateProvider, + useConnectorBuilderState, +} from "services/connectorBuilder/ConnectorBuilderStateService"; import styles from "./ConnectorBuilderPage.module.scss"; const ConnectorBuilderPageInner: React.FC = () => { const { formatMessage } = useIntl(); - const [showYamlEditor, toggleYamlEditor] = useToggle(true); + const [showYamlEditor, toggleYamlEditor] = useToggle(false); + + const { builderFormValues } = useConnectorBuilderState(); return ( - - {showYamlEditor ? ( - - ) : ( - - )} - - ), - className: styles.leftPanel, - minWidth: 100, - }} - secondPanel={{ - children: , - className: styles.rightPanel, - flex: 0.33, - minWidth: 60, - overlay: { - displayThreshold: 325, - header: formatMessage({ id: "connectorBuilder.testConnector" }), - rotation: "counter-clockwise", - }, + { + console.log(values); }} - /> + > + {({ values }) => ( + + {showYamlEditor ? ( + + ) : ( + + )} + + ), + className: styles.leftPanel, + minWidth: 100, + }} + secondPanel={{ + children: , + className: styles.rightPanel, + flex: 0.33, + minWidth: 60, + overlay: { + displayThreshold: 325, + header: formatMessage({ id: "connectorBuilder.testConnector" }), + rotation: "counter-clockwise", + }, + }} + /> + )} + ); }; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 7d61aa766202..a1b11bac4fef 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -3,13 +3,30 @@ import React, { useContext, useEffect, useMemo, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; +import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/Builder/types"; + import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useListStreams } from "./ConnectorBuilderApiService"; +export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { + connectorName: "", + urlBase: "", + streams: [], +}; + +const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { + version: "0.1.0", + check: { + stream_names: [], + }, + streams: [], +}; + interface Context { + builderFormValues: BuilderFormValues; jsonManifest: ConnectorManifest; yamlManifest: string; yamlEditorIsMounted: boolean; @@ -19,6 +36,8 @@ interface Context { selectedStream?: StreamsListReadStreamsItem; configString: string; configJson: StreamReadRequestBodyConfig; + resetBuilderFormValues: () => void; + setBuilderFormValues: (values: BuilderFormValues) => void; setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlManifest: (yamlValue: string) => void; setYamlEditorIsMounted: (value: boolean) => void; @@ -32,19 +51,25 @@ export const ConnectorBuilderStateContext = React.createContext( export const ConnectorBuilderStateProvider: React.FC> = ({ children }) => { const { formatMessage } = useIntl(); - // json manifest - const defaultJsonManifest = { - version: "1.0.0", - check: { - stream_names: [], - }, - streams: [], - }; + const [builderFormValues, setBuilderFormValues] = useLocalStorage( + "connectorBuilderFormValues", + DEFAULT_BUILDER_FORM_VALUES + ); + const formValues = builderFormValues ?? DEFAULT_BUILDER_FORM_VALUES; + const resetBuilderFormValues = () => setBuilderFormValues(DEFAULT_BUILDER_FORM_VALUES); + console.log("formValues", formValues); + const [jsonManifest, setJsonManifest] = useLocalStorage( - "connectorBuilderManifest", - defaultJsonManifest + "connectorBuilderJsonManifest", + DEFAULT_JSON_MANIFEST_VALUES ); - const manifest = jsonManifest ?? defaultJsonManifest; + const manifest = jsonManifest ?? DEFAULT_JSON_MANIFEST_VALUES; + console.log("manifest", manifest); + + useEffect(() => { + setJsonManifest(convertToManifest(formValues)); + }, [formValues, setJsonManifest]); + const [yamlIsValid, setYamlIsValid] = useState(true); const [yamlEditorIsMounted, setYamlEditorIsMounted] = useState(true); @@ -66,7 +91,7 @@ export const ConnectorBuilderStateProvider: React.FC stream.name === selectedStreamName); const ctx = { + builderFormValues: formValues, jsonManifest: manifest, yamlManifest, yamlEditorIsMounted, @@ -108,6 +134,8 @@ export const ConnectorBuilderStateProvider: React.FC Date: Fri, 2 Dec 2022 18:31:09 -0800 Subject: [PATCH 004/100] add modal for adding streams --- .../Builder/AddStreamButton.module.scss | 7 ++ .../Builder/AddStreamButton.tsx | 98 +++++++++++++------ .../connectorBuilder/Builder/BuilderField.tsx | 2 +- .../Builder/BuilderSidebar.tsx | 20 ++-- airbyte-webapp/src/locales/en.json | 4 +- .../ConnectorBuilderStateService.tsx | 3 - 6 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss new file mode 100644 index 000000000000..7bead5293526 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss @@ -0,0 +1,7 @@ +@use "scss/variables"; + +.body { + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 8842b3b1713a..b7d4b1a2c70e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -1,9 +1,12 @@ -import { Formik } from "formik"; +import { Form, Formik, useFormikContext } from "formik"; import { useState } from "react"; +import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; +import { Heading } from "components/ui/Heading"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import styles from "./AddStreamButton.module.scss"; import { FormikPatch } from "./Builder"; import { BuilderField } from "./BuilderField"; @@ -12,38 +15,77 @@ interface AddStreamValues { urlPath: string; } -export const AddStreamButton: React.FC = () => { +interface AddStreamButtonProps { + numStreams: number; +} + +export const AddStreamButton: React.FC = ({ numStreams }) => { const [isOpen, setIsOpen] = useState(false); + const { setFieldValue } = useFormikContext(); return ( <> - + {isOpen && ( - setIsOpen(false)}> - { - console.log(values); - setIsOpen(false); - }} - > - <> - - - - - - - - - - - + { + setFieldValue(`streams[${numStreams}]`, { + name: values.streamName, + urlPath: values.urlPath, + fieldPointer: [], + httpMethod: "GET", + }); + setIsOpen(false); + }} + > + <> + + + + + } + onClose={() => { + setIsOpen(false); + }} + > +
+ + + + + + + + +
+ + + )} ); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 80eef9b28a3c..3d8f7f760b27 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -82,7 +82,7 @@ export const BuilderField: React.FC = ({ path, label, tooltip const [field, meta, helpers] = useField(fieldConfig); const hasError = !!meta.error && meta.touched; - console.log(`path: ${path}, value: ${field.value}`); + // console.log(`path: ${path}, value: ${field.value}, hasError: ${hasError}`); return ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 70a9ef4b8433..c2c4b099c520 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -3,7 +3,10 @@ import { useField, useFormikContext } from "formik"; import { Heading } from "components/ui/Heading"; -import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { + DEFAULT_BUILDER_FORM_VALUES, + useConnectorBuilderState, +} from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; import { AddStreamButton } from "./AddStreamButton"; @@ -39,13 +42,15 @@ export const BuilderSidebar: React.FC = ({ numStreams, onViewSelect, }) => { - const { yamlManifest, resetBuilderFormValues } = useConnectorBuilderState(); - const { resetForm } = useFormikContext(); + const { yamlManifest } = useConnectorBuilderState(); + const { setValues } = useFormikContext(); const handleResetForm = () => { - resetBuilderFormValues(); - resetForm(); + setValues(DEFAULT_BUILDER_FORM_VALUES); + onViewSelect("global"); }; + console.log("numStreams", numStreams); + return (
@@ -53,14 +58,14 @@ export const BuilderSidebar: React.FC = ({ Connector Name - + Streams - + {Array.from(Array(numStreams).keys()).map((streamNum) => { const streamPath = `streams[${streamNum}]`; @@ -68,7 +73,6 @@ export const BuilderSidebar: React.FC = ({ })} -
); }; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 2204b83338e3..85090035771e 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -59,6 +59,7 @@ "form.delete": "Delete", "form.change": "Change", "form.add": "Add", + "form.create": "Create", "form.saveChanges": "Save changes", "form.saveChangesAndTest": "Save changes and test", "form.sourceRetest": "Retest source", @@ -588,5 +589,6 @@ "connectorBuilder.unknownError": "An unknown error has occurred", "connectorBuilder.testConnector": "TEST YOUR CONNECTOR", "connectorBuilder.couldNotDetectStreams": "Could not detect streams in the YAML editor:", - "connectorBuilder.ensureProperYaml": "In order to test a stream, ensure that the YAML is structured as described in the docs." + "connectorBuilder.ensureProperYaml": "In order to test a stream, ensure that the YAML is structured as described in the docs.", + "connectorBuilder.newStream": "New stream" } diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index a1b11bac4fef..a89b324fe320 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -36,7 +36,6 @@ interface Context { selectedStream?: StreamsListReadStreamsItem; configString: string; configJson: StreamReadRequestBodyConfig; - resetBuilderFormValues: () => void; setBuilderFormValues: (values: BuilderFormValues) => void; setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlManifest: (yamlValue: string) => void; @@ -56,7 +55,6 @@ export const ConnectorBuilderStateProvider: React.FC setBuilderFormValues(DEFAULT_BUILDER_FORM_VALUES); console.log("formValues", formValues); const [jsonManifest, setJsonManifest] = useLocalStorage( @@ -135,7 +133,6 @@ export const ConnectorBuilderStateProvider: React.FC Date: Mon, 5 Dec 2022 09:59:10 -0800 Subject: [PATCH 005/100] focus stream after adding and reset button style --- .../connectorBuilder/Builder/AddStreamButton.tsx | 4 +++- .../Builder/BuilderSidebar.module.scss | 5 +++++ .../connectorBuilder/Builder/BuilderSidebar.tsx | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index b7d4b1a2c70e..71c3225cc9fe 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -17,9 +17,10 @@ interface AddStreamValues { interface AddStreamButtonProps { numStreams: number; + onAddStream: (addedStreamNum: number) => void; } -export const AddStreamButton: React.FC = ({ numStreams }) => { +export const AddStreamButton: React.FC = ({ numStreams, onAddStream }) => { const [isOpen, setIsOpen] = useState(false); const { setFieldValue } = useFormikContext(); @@ -43,6 +44,7 @@ export const AddStreamButton: React.FC = ({ numStreams }) httpMethod: "GET", }); setIsOpen(false); + onAddStream(numStreams); }} > <> diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 3fad3fbfabfb..0d95f2a1f1b7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -30,6 +30,11 @@ .downloadButton { margin-top: auto; + width: 85%; +} + +.resetButton { + margin-top: variables.$spacing-sm; margin-bottom: variables.$spacing-md; width: 85%; } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index c2c4b099c520..2864ca9e084f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,6 +1,9 @@ +import { faRotateLeft } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useField, useFormikContext } from "formik"; +import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; import { @@ -58,14 +61,13 @@ export const BuilderSidebar: React.FC = ({ Connector Name - Streams - + onViewSelect(addedStreamNum)} /> {Array.from(Array(numStreams).keys()).map((streamNum) => { const streamPath = `streams[${streamNum}]`; @@ -73,6 +75,14 @@ export const BuilderSidebar: React.FC = ({ })} +
); }; From 2352ef9ed0bab9fede5e5d2fd9a6e33e83798dcf Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 5 Dec 2022 16:29:02 -0800 Subject: [PATCH 006/100] add reset confirm modal and select view on add --- .../Builder/AddStreamButton.tsx | 4 +-- .../connectorBuilder/Builder/Builder.tsx | 15 +++++--- .../Builder/BuilderSidebar.tsx | 36 ++++++++++++------- airbyte-webapp/src/locales/en.json | 5 ++- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 71c3225cc9fe..a56e7eeb303f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -17,7 +17,7 @@ interface AddStreamValues { interface AddStreamButtonProps { numStreams: number; - onAddStream: (addedStreamNum: number) => void; + onAddStream: (addedStreamNum: number, addedStreamName: string) => void; } export const AddStreamButton: React.FC = ({ numStreams, onAddStream }) => { @@ -44,7 +44,7 @@ export const AddStreamButton: React.FC = ({ numStreams, on httpMethod: "GET", }); setIsOpen(false); - onAddStream(numStreams); + onAddStream(numStreams, values.streamName); }} > <> diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index f2640ef42d32..d199a17f0aa1 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -5,7 +5,7 @@ import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBui import { usePatchFormik } from "views/Connector/ConnectorForm/useBuildForm"; import styles from "./Builder.module.scss"; -import { BuilderSidebar } from "./BuilderSidebar"; +import { BuilderSidebar, BuilderView } from "./BuilderSidebar"; import { GlobalConfigView } from "./GlobalConfigView"; import { StreamConfigView } from "./StreamConfigView"; import { BuilderFormValues } from "./types"; @@ -21,12 +21,19 @@ interface BuilderProps { } export const Builder: React.FC = ({ values, toggleYamlEditor }) => { - const { setBuilderFormValues } = useConnectorBuilderState(); + const { setBuilderFormValues, setSelectedStream } = useConnectorBuilderState(); useEffect(() => { setBuilderFormValues(values); }, [values, setBuilderFormValues]); - const [selectedView, setSelectedView] = useState<"global" | number>("global"); + const [selectedView, setSelectedView] = useState("global"); + + const handleViewSelect = (selectedView: BuilderView, streamName?: string) => { + setSelectedView(selectedView); + if (selectedView !== "global" && streamName !== undefined) { + setSelectedStream(streamName); + } + }; console.log("values", values); @@ -38,7 +45,7 @@ export const Builder: React.FC = ({ values, toggleYamlEditor }) => className={styles.sidebar} toggleYamlEditor={toggleYamlEditor} numStreams={values.streams.length} - onViewSelect={setSelectedView} + onViewSelect={handleViewSelect} />
{selectedView === "global" ? : } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 2864ca9e084f..034a60c6764a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -6,6 +6,7 @@ import { useField, useFormikContext } from "formik"; import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { DEFAULT_BUILDER_FORM_VALUES, useConnectorBuilderState, @@ -19,24 +20,22 @@ import { UiYamlToggleButton } from "./UiYamlToggleButton"; export type BuilderView = "global" | number; interface StreamSelectButtonProps { - streamPath: string; - onClick: () => void; + streamNum: number; + onSelectStream: (streamNum: number, streamName: string) => void; } -const StreamSelectButton: React.FC = ({ streamPath, onClick }) => { - const streamNamePath = `${streamPath}.name`; - console.log("streamNamePath", streamNamePath); +const StreamSelectButton: React.FC = ({ streamNum, onSelectStream }) => { + const streamNamePath = `streams[${streamNum}].name`; const [field] = useField(streamNamePath); - console.log("field.value", field.value); - return ; + return ; }; interface BuilderSidebarProps { className?: string; toggleYamlEditor: () => void; numStreams: number; - onViewSelect: (selected: BuilderView) => void; + onViewSelect: (selected: BuilderView, streamName?: string) => void; } export const BuilderSidebar: React.FC = ({ @@ -45,11 +44,20 @@ export const BuilderSidebar: React.FC = ({ numStreams, onViewSelect, }) => { + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { yamlManifest } = useConnectorBuilderState(); const { setValues } = useFormikContext(); const handleResetForm = () => { - setValues(DEFAULT_BUILDER_FORM_VALUES); - onViewSelect("global"); + openConfirmationModal({ + text: "connectorBuilder.resetModal.text", + title: "connectorBuilder.resetModal.title", + submitButtonText: "connectorBuilder.resetModal.submitButton", + onSubmit: () => { + setValues(DEFAULT_BUILDER_FORM_VALUES); + onViewSelect("global"); + closeConfirmationModal(); + }, + }); }; console.log("numStreams", numStreams); @@ -67,11 +75,13 @@ export const BuilderSidebar: React.FC = ({ Streams - onViewSelect(addedStreamNum)} /> + onViewSelect(addedStreamNum, addedStreamName)} + /> {Array.from(Array(numStreams).keys()).map((streamNum) => { - const streamPath = `streams[${streamNum}]`; - return onViewSelect(streamNum)} />; + return ; })} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 85090035771e..9ec83da89dab 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -590,5 +590,8 @@ "connectorBuilder.testConnector": "TEST YOUR CONNECTOR", "connectorBuilder.couldNotDetectStreams": "Could not detect streams in the YAML editor:", "connectorBuilder.ensureProperYaml": "In order to test a stream, ensure that the YAML is structured as described in the docs.", - "connectorBuilder.newStream": "New stream" + "connectorBuilder.newStream": "New stream", + "connectorBuilder.resetModal.text": "This will erase all streams and values that are currently set. Are you sure you want to continue?", + "connectorBuilder.resetModal.title": "Reset Connector Builder", + "connectorBuilder.resetModal.submitButton": "Reset" } From df492fadb67fcd8132933fa51c89b5be18c7ab2c Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 5 Dec 2022 18:00:53 -0800 Subject: [PATCH 007/100] style global config and streams buttons --- .../Builder/AddStreamButton.module.scss | 16 +++++ .../Builder/AddStreamButton.tsx | 13 ++-- .../connectorBuilder/Builder/Builder.tsx | 1 + .../Builder/BuilderSidebar.module.scss | 63 +++++++++++++++++- .../Builder/BuilderSidebar.tsx | 66 ++++++++++++++----- airbyte-webapp/src/locales/en.json | 4 +- 6 files changed, 139 insertions(+), 24 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss index 7bead5293526..9cb5426473d3 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss @@ -1,7 +1,23 @@ @use "scss/variables"; +@use "scss/colors"; .body { display: flex; flex-direction: column; gap: variables.$spacing-xl; } + +.addButton { + width: 26px; + height: 26px !important; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + padding: 9px !important; +} + +.plus { + height: 30px; + width: 30px; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index a56e7eeb303f..95a23cf7a4a8 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -1,3 +1,4 @@ +import classNames from "classnames"; import { Form, Formik, useFormikContext } from "formik"; import { useState } from "react"; import { FormattedMessage } from "react-intl"; @@ -6,6 +7,7 @@ import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg"; import styles from "./AddStreamButton.module.scss"; import { FormikPatch } from "./Builder"; import { BuilderField } from "./BuilderField"; @@ -16,23 +18,24 @@ interface AddStreamValues { } interface AddStreamButtonProps { + className?: string; numStreams: number; onAddStream: (addedStreamNum: number, addedStreamName: string) => void; } -export const AddStreamButton: React.FC = ({ numStreams, onAddStream }) => { +export const AddStreamButton: React.FC = ({ className, numStreams, onAddStream }) => { const [isOpen, setIsOpen] = useState(false); const { setFieldValue } = useFormikContext(); return ( <> - + icon={} + /> {isOpen && ( = ({ values, toggleYamlEditor }) => toggleYamlEditor={toggleYamlEditor} numStreams={values.streams.length} onViewSelect={handleViewSelect} + selectedView={selectedView} /> {selectedView === "global" ? : } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 0d95f2a1f1b7..fb00ee8f23e8 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -2,7 +2,7 @@ @use "scss/colors"; .container { - // padding: variables.$spacing-sm; + padding: variables.$spacing-sm; display: flex; flex-direction: column; align-items: center; @@ -28,13 +28,70 @@ justify-content: center; } +.streamsHeader { + display: flex; + align-items: center; + width: 100%; + padding: 0; + margin-top: 30px; +} + +.streamsHeading { + margin-left: variables.$spacing-md; + color: colors.$blue; +} + +.addStreamButton { + margin-left: auto; + margin-right: variables.$spacing-sm; +} + .downloadButton { margin-top: auto; - width: 85%; + width: 90%; } .resetButton { margin-top: variables.$spacing-sm; margin-bottom: variables.$spacing-md; - width: 85%; + width: 90%; +} + +.viewButton { + border: none; + width: 100%; + height: 34px; + cursor: pointer; + border-radius: variables.$border-radius-md; + padding: 9px 10px; + display: flex; + align-items: center; + gap: 5px; +} + +.unselectedViewButton { + background-color: transparent; + color: colors.$dark-blue; + + &:hover { + background-color: colors.$grey-50; + } +} + +.selectedViewButton { + background-color: colors.$dark-blue; + color: colors.$white; +} + +.globalConfigButton { + margin-top: 20px; +} + +.streamList { + width: 100%; + margin-top: 10px; + margin-bottom: 10px; + display: flex; + flex-direction: column; + overflow-y: auto; } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 034a60c6764a..04b1f312f38d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,10 +1,12 @@ -import { faRotateLeft } from "@fortawesome/free-solid-svg-icons"; +import { faRotateLeft, faSliders } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useField, useFormikContext } from "formik"; +import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; +import { Text } from "components/ui/Text"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { @@ -22,13 +24,24 @@ export type BuilderView = "global" | number; interface StreamSelectButtonProps { streamNum: number; onSelectStream: (streamNum: number, streamName: string) => void; + selected: boolean; } -const StreamSelectButton: React.FC = ({ streamNum, onSelectStream }) => { +const StreamSelectButton: React.FC = ({ streamNum, onSelectStream, selected }) => { const streamNamePath = `streams[${streamNum}].name`; const [field] = useField(streamNamePath); - return ; + return ( + + ); }; interface BuilderSidebarProps { @@ -36,6 +49,7 @@ interface BuilderSidebarProps { toggleYamlEditor: () => void; numStreams: number; onViewSelect: (selected: BuilderView, streamName?: string) => void; + selectedView: BuilderView; } export const BuilderSidebar: React.FC = ({ @@ -43,6 +57,7 @@ export const BuilderSidebar: React.FC = ({ toggleYamlEditor, numStreams, onViewSelect, + selectedView, }) => { const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { yamlManifest } = useConnectorBuilderState(); @@ -69,25 +84,46 @@ export const BuilderSidebar: React.FC = ({ Connector Name - + - - Streams - +
+ + + - onViewSelect(addedStreamNum, addedStreamName)} - /> + onViewSelect(addedStreamNum, addedStreamName)} + /> +
- {Array.from(Array(numStreams).keys()).map((streamNum) => { - return ; - })} +
+ {Array.from(Array(numStreams).keys()).map((streamNum) => { + return ( + + ); + })} +
); }; +interface StreamSelectButtonProps { + streamNum: number; + onSelectStream: (streamNum: number, streamName: string) => void; + selected: boolean; +} + +const StreamSelectButton: React.FC = ({ streamNum, onSelectStream, selected }) => { + const streamPath = `streams[${streamNum}]`; + const [field] = useField(`${streamPath}.name`); + + return ( + onSelectStream(streamNum, field.value)}> + {field.value} + + ); +}; + interface BuilderSidebarProps { className?: string; toggleYamlEditor: () => void; @@ -89,16 +108,14 @@ export const BuilderSidebar: React.FC = ({
- +
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 49c060dc7617..a6b5524f288d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -5,10 +5,15 @@ export const GlobalConfigView: React.FC = () => { return ( <> - + - + ); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts b/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts index 963821026735..a333c3655c3f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts @@ -5,10 +5,12 @@ import { } from "core/request/ConnectorBuilderClient"; export interface BuilderFormValues { - connectorName: string; - urlBase: string; - // TODO: make required when authenticator is fully added - authenticator?: HttpRequesterAllOfAuthenticator; + global: { + connectorName: string; + urlBase: string; + // TODO: make required when authenticator is fully added + authenticator?: HttpRequesterAllOfAuthenticator; + }; streams: BuilderStream[]; } @@ -27,9 +29,9 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest name: stream.name, requester: { name: stream.name, - url_base: values.urlBase, + url_base: values.global?.urlBase, path: stream.urlPath, - authenticator: values.authenticator, + authenticator: values.global?.authenticator, // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema config: {}, }, diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx index 7beb1f1d6934..9b7e3af62c1e 100644 --- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx +++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx @@ -48,7 +48,7 @@ const ConnectorBuilderPageInner: React.FC = () => { secondPanel={{ children: , className: styles.rightPanel, - flex: 0, + flex: 0.33, minWidth: 60, overlay: { displayThreshold: 325, diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 2c3f29ce2b87..70f0227b2cae 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -15,8 +15,10 @@ import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useListStreams } from "./ConnectorBuilderApiService"; export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { - connectorName: "", - urlBase: "", + global: { + connectorName: "", + urlBase: "", + }, streams: [], }; From 9bb1812de38451acff5c5a77d0a9ffe2ee061818 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 6 Dec 2022 18:13:06 -0800 Subject: [PATCH 018/100] confirmation modal on toggling dirty form + cleanup --- .../connectorBuilder/Builder/Builder.tsx | 2 - .../Builder/BuilderSidebar.tsx | 2 - .../StreamTestingPanel/StreamTestingPanel.tsx | 1 - .../YamlEditor/YamlEditor.tsx | 43 ++++++++++++++++--- airbyte-webapp/src/locales/en.json | 3 ++ .../ConnectorBuilderStateService.tsx | 5 +-- 6 files changed, 40 insertions(+), 16 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index ed3b2b3667dc..f9be2593e8c4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -35,8 +35,6 @@ export const Builder: React.FC = ({ values, toggleYamlEditor }) => } }; - console.log("values", values); - return ( <> diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 8f087168c6af..af6188951dbf 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -96,8 +96,6 @@ export const BuilderSidebar: React.FC = ({ const [field] = useField("connectorName"); - console.log("numStreams", numStreams); - return (
diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx index 299fc256c57f..3d62a8e5e413 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx @@ -26,7 +26,6 @@ export const StreamTestingPanel: React.FC = () => { } const hasStreams = jsonManifest.streams?.length > 0; - console.log(hasStreams); return (
diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 94aa2a0eb92b..69efcb647698 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -1,13 +1,16 @@ import { useMonaco } from "@monaco-editor/react"; import { load, YAMLException } from "js-yaml"; +import isMatch from "lodash/isMatch"; import { editor } from "monaco-editor/esm/vs/editor/editor.api"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { CodeEditor } from "components/ui/CodeEditor"; import { ConnectorManifest } from "core/request/ConnectorBuilderClient"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { convertToManifest } from "../Builder/types"; import { UiYamlToggleButton } from "../Builder/UiYamlToggleButton"; import { DownloadYamlButton } from "./DownloadYamlButton"; import styles from "./YamlEditor.module.scss"; @@ -17,12 +20,17 @@ interface YamlEditorProps { } export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const yamlEditorRef = useRef(); - // const template = useManifestTemplate(); - // const [locallyStoredYaml, setLocallyStoredYaml] = useLocalStorage("connectorBuilderYaml", template); - // useDebounce(() => setLocallyStoredYaml(yamlValue), 500, [yamlValue]); - const { yamlManifest, yamlIsValid, setYamlEditorIsMounted, setYamlIsValid, setJsonManifest } = - useConnectorBuilderState(); + const { + yamlManifest, + yamlIsValid, + jsonManifest, + builderFormValues, + setYamlEditorIsMounted, + setYamlIsValid, + setJsonManifest, + } = useConnectorBuilderState(); const [yamlValue, setYamlValue] = useState(yamlManifest); const monaco = useMonaco(); @@ -64,10 +72,31 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { } }, [yamlValue, monaco, setJsonManifest, setYamlIsValid]); + const yamlIsDirty = useMemo(() => { + return !isMatch(convertToManifest(builderFormValues), jsonManifest); + }, [jsonManifest, builderFormValues]); + console.log(yamlIsDirty); + + const handleToggleYamlEditor = () => { + if (yamlIsDirty) { + openConfirmationModal({ + text: "connectorBuilder.toggleModal.text", + title: "connectorBuilder.toggleModal.title", + submitButtonText: "connectorBuilder.toggleModal.submitButton", + onSubmit: () => { + toggleYamlEditor(); + closeConfirmationModal(); + }, + }); + } else { + toggleYamlEditor(); + } + }; + return (
- +
diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 3f0835c49137..64b0e014547c 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -603,6 +603,9 @@ "connectorBuilder.streamsHeading": "STREAMS ({number})", "connectorBuilder.globalConfiguration": "Global Configuration", "connectorBuilder.noStreamsMessage": "Add a stream to test it here", + "connectorBuilder.toggleModal.text": "Toggling back to the UI will erase any changes you have made in the YAML editor.\n\nIn order to export your current yaml, click the Download YAML button.", + "connectorBuilder.toggleModal.title": "Warning", + "connectorBuilder.toggleModal.submitButton": "Confirm", "cloudApi.loginCallbackUrlError": "There was an error connecting to the developer portal. Please try again." } diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 70f0227b2cae..e109c1b92c55 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -55,19 +55,18 @@ export const ConnectorBuilderStateContext = React.createContext( export const ConnectorBuilderStateProvider: React.FC> = ({ children }) => { const { formatMessage } = useIntl(); + // manifest values const [builderFormValues, setBuilderFormValues] = useLocalStorage( "connectorBuilderFormValues", DEFAULT_BUILDER_FORM_VALUES ); const formValues = builderFormValues ?? DEFAULT_BUILDER_FORM_VALUES; - console.log("formValues", formValues); const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", DEFAULT_JSON_MANIFEST_VALUES ); const manifest = jsonManifest ?? DEFAULT_JSON_MANIFEST_VALUES; - console.log("manifest", manifest); useEffect(() => { setJsonManifest(convertToManifest(formValues)); @@ -94,8 +93,6 @@ export const ConnectorBuilderStateProvider: React.FC Date: Tue, 6 Dec 2022 18:26:13 -0800 Subject: [PATCH 019/100] fix connector name display --- .../src/components/connectorBuilder/Builder/BuilderSidebar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index af6188951dbf..60837dfdc53a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -94,7 +94,7 @@ export const BuilderSidebar: React.FC = ({ }); }; - const [field] = useField("connectorName"); + const [field] = useField("global.connectorName"); return (
From 094a045ddeca70acefdf510f500e3bb6d203e197 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 6 Dec 2022 18:31:11 -0800 Subject: [PATCH 020/100] undo change to manifest schema --- .../sources/declarative/config_component_schema.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index c15f3f94902a..8f61c0716113 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -86,6 +86,16 @@ "type": "string", "default": "" }, + "_schema_loader": { + "anyOf": [ + { + "$ref": "#/definitions/JsonFileSchemaLoader" + }, + { + "$ref": "#/definitions/DefaultSchemaLoader" + } + ] + }, "stream_cursor_field": { "anyOf": [ { From d1c8c800640204d2c9788a00bf0d6bb70c97ffa5 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 6 Dec 2022 18:37:01 -0800 Subject: [PATCH 021/100] remove commented code --- .../connectorBuilder/Builder/BuilderField.module.scss | 6 ------ .../connectorBuilder/Builder/BuilderField.tsx | 10 ---------- .../StreamTestingPanel/StreamSelector.module.scss | 2 -- .../connectorBuilder/YamlEditor/YamlEditor.module.scss | 2 -- .../connectorBuilder/ConnectorBuilderStateService.tsx | 2 -- 5 files changed, 22 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss index 332592b3a616..f198f89561f5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss @@ -1,12 +1,6 @@ @use "scss/variables"; @use "scss/colors"; -// .container { -// display: flex; -// flex-direction: column; -// gap: 0; -// } - .error { margin-top: variables.$spacing-sm; color: colors.$red; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 3d8f7f760b27..bd799ff8efed 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -35,10 +35,6 @@ interface BaseFieldProps { type BuilderFieldProps = BaseFieldProps & ({ type: "text" } | { type: "array" } | { type: "enum"; options: string[] }); const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { - // useEffect(() => { - // setValue(value); - // }, []); // eslint-disable-line react-hooks/exhaustive-deps - return ( = ({ options, value, setValue, error, }; const ArrayField: React.FC = ({ name, value, setValue, error }) => { - // useEffect(() => { - // setValue(value); - // }, []); // eslint-disable-line react-hooks/exhaustive-deps - return setValue(value)} error={error} />; }; @@ -82,8 +74,6 @@ export const BuilderField: React.FC = ({ path, label, tooltip const [field, meta, helpers] = useField(fieldConfig); const hasError = !!meta.error && meta.touched; - // console.log(`path: ${path}, value: ${field.value}, hasError: ${hasError}`); - return ( {props.type === "text" && } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss index 8c949a0f5a60..c60827f96cdc 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss @@ -14,8 +14,6 @@ border-radius: variables.$border-radius-md; border: none; display: flex; - - // justify-content: center; } .label { diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss index 0206dfd09128..6b0e54f4d0e5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.module.scss @@ -14,8 +14,6 @@ display: flex; padding: variables.$spacing-md; flex-direction: row; - - // justify-content: flex-end; } .editorContainer { diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index aa4aea7e9937..b0696ad8cb05 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -107,8 +107,6 @@ export const ConnectorBuilderStateProvider: React.FC { return streamListRead?.streams ?? []; }, [streamListRead]); - // const streamListErrorMessage = undefined; - // const streams = useMemo(() => [{ name: "stream1", url: "url1" }], []); const firstStreamName = streams.length > 0 ? streams[0].name : undefined; const [selectedStreamName, setSelectedStream] = useState(firstStreamName); From aebac20fe74ec14929479f4af93b875b3988fa6a Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 6 Dec 2022 18:42:29 -0800 Subject: [PATCH 022/100] remove unnecessary change --- .../components/ui/ResizablePanels/ResizablePanels.module.scss | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.module.scss b/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.module.scss index 35f099d5203d..d73b158f15c9 100644 --- a/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.module.scss +++ b/airbyte-webapp/src/components/ui/ResizablePanels/ResizablePanels.module.scss @@ -1,6 +1,5 @@ @use "scss/colors"; @use "scss/variables"; -@use "scss/mixins"; /* * This eliminates flickering scrollbars when resizing using the reflex splitter. From 1e39be45503a022934dfaaee4c71433dcdcd74a7 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 7 Dec 2022 15:04:03 -0800 Subject: [PATCH 023/100] fix spacing --- .../connectorBuilder/Builder/BuilderSidebar.module.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index c1da069e93aa..07875d9ce7b6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -36,7 +36,7 @@ align-items: center; width: 100%; padding: 0; - margin-top: variables.$spacing-xl; + margin-top: variables.$spacing-2xl; } .streamsHeading { From 09cf87565b9db791230a014906f1cd867d51c92c Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 7 Dec 2022 15:05:26 -0800 Subject: [PATCH 024/100] use shadow mixin for connector img --- .../connectorBuilder/Builder/BuilderSidebar.module.scss | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 07875d9ce7b6..f5b9ab771c84 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -1,5 +1,6 @@ @use "scss/variables"; @use "scss/colors"; +@use "scss/mixins"; .container { padding: variables.$spacing-sm; @@ -10,8 +11,9 @@ } .connectorImg { + @include mixins.shadow; + width: 90px; - box-shadow: 0 2px 4px rgba(26, 25, 77, 12%); border-radius: 25px; padding: variables.$spacing-xl; margin-top: 50px; From ec65d67ca5d7146212af3759cb1256845dac155a Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 7 Dec 2022 15:06:25 -0800 Subject: [PATCH 025/100] add comment about connector img --- .../src/components/connectorBuilder/Builder/BuilderSidebar.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 60837dfdc53a..aa8a62aee776 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -99,7 +99,10 @@ export const BuilderSidebar: React.FC = ({ return (
+ + {/* TODO: replace with uploaded img when that functionality is added */} Connector Logo +
{field.value} From 603820acb75b2db2db6984cf006068198f2c8b94 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 11:35:59 -0800 Subject: [PATCH 026/100] change onSubmit to no-op --- .../pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx index 9b7e3af62c1e..cb648dcd5cae 100644 --- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx +++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx @@ -4,7 +4,6 @@ import { useIntl } from "react-intl"; import { useToggle } from "react-use"; import { Builder } from "components/connectorBuilder/Builder/Builder"; -import { BuilderFormValues } from "components/connectorBuilder/Builder/types"; import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel"; import { YamlEditor } from "components/connectorBuilder/YamlEditor"; import { ResizablePanels } from "components/ui/ResizablePanels"; @@ -23,12 +22,7 @@ const ConnectorBuilderPageInner: React.FC = () => { const { builderFormValues } = useConnectorBuilderState(); return ( - { - console.log(values); - }} - > + undefined}> {({ values }) => ( Date: Thu, 8 Dec 2022 11:36:30 -0800 Subject: [PATCH 027/100] remove console log --- .../src/components/connectorBuilder/YamlEditor/YamlEditor.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 69efcb647698..9f7721b409c6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -75,7 +75,6 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { const yamlIsDirty = useMemo(() => { return !isMatch(convertToManifest(builderFormValues), jsonManifest); }, [jsonManifest, builderFormValues]); - console.log(yamlIsDirty); const handleToggleYamlEditor = () => { if (yamlIsDirty) { From d1672a0c92901de9507e964a0ba1e334adccd7a1 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 11:42:26 -0800 Subject: [PATCH 028/100] clean up styling --- .../Builder/AddStreamButton.module.scss | 11 ++++------- .../connectorBuilder/Builder/AddStreamButton.tsx | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss index 9cb5426473d3..6c1f9c41b990 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss @@ -1,6 +1,8 @@ @use "scss/variables"; @use "scss/colors"; +$buttonWidth: 26px; + .body { display: flex; flex-direction: column; @@ -8,16 +10,11 @@ } .addButton { - width: 26px; - height: 26px !important; + width: $buttonWidth; + height: $buttonWidth !important; border-radius: 50%; display: flex; align-items: center; justify-content: center; padding: 9px !important; } - -.plus { - height: 30px; - width: 30px; -} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 09e46b9a1125..4644d14c04c7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -33,7 +33,7 @@ export const AddStreamButton: React.FC = ({ className, num onClick={() => { setIsOpen(true); }} - icon={} + icon={} /> {isOpen && ( Date: Thu, 8 Dec 2022 11:55:15 -0800 Subject: [PATCH 029/100] simplify sidebar to remove StreamSelectButton component --- .../Builder/BuilderSidebar.tsx | 41 ++++--------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index aa8a62aee776..57411a780151 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,7 +1,7 @@ import { faRotateLeft, faSliders } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; -import { useField, useFormikContext } from "formik"; +import { useFormikContext } from "formik"; import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; @@ -17,6 +17,7 @@ import { import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; +import { BuilderFormValues } from "./types"; import { UiYamlToggleButton } from "./UiYamlToggleButton"; export type BuilderView = "global" | number; @@ -46,23 +47,6 @@ const ViewSelectButton: React.FC> ); }; -interface StreamSelectButtonProps { - streamNum: number; - onSelectStream: (streamNum: number, streamName: string) => void; - selected: boolean; -} - -const StreamSelectButton: React.FC = ({ streamNum, onSelectStream, selected }) => { - const streamPath = `streams[${streamNum}]`; - const [field] = useField(`${streamPath}.name`); - - return ( - onSelectStream(streamNum, field.value)}> - {field.value} - - ); -}; - interface BuilderSidebarProps { className?: string; toggleYamlEditor: () => void; @@ -80,7 +64,7 @@ export const BuilderSidebar: React.FC = ({ }) => { const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { yamlManifest } = useConnectorBuilderState(); - const { setValues } = useFormikContext(); + const { values, setValues } = useFormikContext(); const handleResetForm = () => { openConfirmationModal({ text: "connectorBuilder.resetModal.text", @@ -94,8 +78,6 @@ export const BuilderSidebar: React.FC = ({ }); }; - const [field] = useField("global.connectorName"); - return (
@@ -105,7 +87,7 @@ export const BuilderSidebar: React.FC = ({
- {field.value} + {values.global?.connectorName}
@@ -131,16 +113,11 @@ export const BuilderSidebar: React.FC = ({
- {Array.from(Array(numStreams).keys()).map((streamNum) => { - return ( - - ); - })} + {values.streams.map(({ name }, num) => ( + onViewSelect(num, name)}> + {name} + + ))}
From 65cd42b2694d43b3aa7170c219428750bd1f5ebd Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 11:57:33 -0800 Subject: [PATCH 030/100] swap colors of toggle --- .../Builder/UiYamlToggleButton.module.scss | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss index 9e18e7554dd0..c8e98efcf62c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss @@ -26,11 +26,11 @@ } .selected { - background-color: colors.$white; - color: colors.$dark-blue; + background-color: transparent; + color: colors.$white; } .unselected { - background-color: transparent; - color: colors.$white; + background-color: colors.$white; + color: colors.$dark-blue; } From 10d789368698558cef2d617ed1bdbadcabe01924 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 13:56:00 -0800 Subject: [PATCH 031/100] move FormikPatch to src/core/form --- .../Builder/AddStreamButton.tsx | 3 +- .../connectorBuilder/Builder/Builder.tsx | 33 +++++++------------ airbyte-webapp/src/core/form/FormikPatch.ts | 29 ++++++++++++++++ .../Connector/ConnectorForm/ConnectorForm.tsx | 7 +--- .../Connector/ConnectorForm/useBuildForm.tsx | 28 +--------------- 5 files changed, 45 insertions(+), 55 deletions(-) create mode 100644 airbyte-webapp/src/core/form/FormikPatch.ts diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 4644d14c04c7..b7955a771f3a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -6,9 +6,10 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { FormikPatch } from "core/form/FormikPatch"; + import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg"; import styles from "./AddStreamButton.module.scss"; -import { FormikPatch } from "./Builder"; import { BuilderField } from "./BuilderField"; interface AddStreamValues { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index f9be2593e8c4..a60d4c6c495a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -2,7 +2,6 @@ import { Form } from "formik"; import { useEffect, useState } from "react"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; -import { usePatchFormik } from "views/Connector/ConnectorForm/useBuildForm"; import styles from "./Builder.module.scss"; import { BuilderSidebar, BuilderView } from "./BuilderSidebar"; @@ -10,11 +9,6 @@ import { GlobalConfigView } from "./GlobalConfigView"; import { StreamConfigView } from "./StreamConfigView"; import { BuilderFormValues } from "./types"; -export const FormikPatch: React.FC = () => { - usePatchFormik(); - return null; -}; - interface BuilderProps { values: BuilderFormValues; toggleYamlEditor: () => void; @@ -36,20 +30,17 @@ export const Builder: React.FC = ({ values, toggleYamlEditor }) => }; return ( - <> - -
- - - {selectedView === "global" ? : } - -
- +
+ +
+ {selectedView === "global" ? : } + +
); }; diff --git a/airbyte-webapp/src/core/form/FormikPatch.ts b/airbyte-webapp/src/core/form/FormikPatch.ts new file mode 100644 index 000000000000..37eee6eb0c95 --- /dev/null +++ b/airbyte-webapp/src/core/form/FormikPatch.ts @@ -0,0 +1,29 @@ +import { flatten } from "flat"; +import { useFormikContext } from "formik"; +import { useEffect } from "react"; + +export const FormikPatch: React.FC = () => { + const { setFieldTouched, isSubmitting, isValidating, errors } = useFormikContext(); + + /* Fixes issue https://github.com/airbytehq/airbyte/issues/1978 + Problem described here https://github.com/formium/formik/issues/445 + The problem is next: + + When we touch the field, it would be set as touched field correctly. + If validation fails on submit - Formik detects touched object mapping based + either on initialValues passed to Formik or on current value set. + So in case of creation, if we touch an input, don't change value and + press submit - our touched map will be cleared. + + This hack just touches all fields on submit. + */ + useEffect(() => { + if (isSubmitting && !isValidating) { + for (const path of Object.keys(flatten(errors))) { + setFieldTouched(path, true, false); + } + } + }, [errors, isSubmitting, isValidating, setFieldTouched]); + + return null; +}; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index 5c3d739bf8b5..028b1faabf29 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -6,6 +6,7 @@ import { useDeepCompareEffect } from "react-use"; import { FormChangeTracker } from "components/common/FormChangeTracker"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; +import { FormikPatch } from "core/form/FormikPatch"; import { FormBaseItem, FormComponentOverrideProps } from "core/form/types"; import { CheckConnectionRead } from "core/request/AirbyteClient"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; @@ -20,14 +21,8 @@ import { useBuildInitialSchema, useBuildUiWidgetsContext, useConstructValidationSchema, - usePatchFormik, } from "./useBuildForm"; -const FormikPatch: React.FC = () => { - usePatchFormik(); - return null; -}; - /** * This function sets all initial const values in the form to current values * @param schema diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index 1f41f7ef850e..5dbea5c17090 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -1,8 +1,6 @@ -import flatten from "flat"; -import { useFormikContext } from "formik"; import { JSONSchema7, JSONSchema7Definition } from "json-schema"; import merge from "lodash/merge"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useMemo, useState } from "react"; import { AnySchema } from "yup"; import { ConnectorDefinitionSpecification } from "core/domain/connector"; @@ -88,27 +86,3 @@ export const useBuildUiWidgetsContext = ( // As validation schema depends on what path of oneOf is currently selected in jsonschema export const useConstructValidationSchema = (jsonSchema: JSONSchema7, uiWidgetsInfo: WidgetConfigMap): AnySchema => useMemo(() => buildYupFormForJsonSchema(jsonSchema, uiWidgetsInfo), [uiWidgetsInfo, jsonSchema]); - -export const usePatchFormik = (): void => { - const { setFieldTouched, isSubmitting, isValidating, errors } = useFormikContext(); - - /* Fixes issue https://github.com/airbytehq/airbyte/issues/1978 - Problem described here https://github.com/formium/formik/issues/445 - The problem is next: - - When we touch the field, it would be set as touched field correctly. - If validation fails on submit - Formik detects touched object mapping based - either on initialValues passed to Formik or on current value set. - So in case of creation, if we touch an input, don't change value and - press submit - our touched map will be cleared. - - This hack just touches all fields on submit. - */ - useEffect(() => { - if (isSubmitting && !isValidating) { - for (const path of Object.keys(flatten(errors))) { - setFieldTouched(path, true, false); - } - } - }, [errors, isSubmitting, isValidating, setFieldTouched]); -}; From fc890017730d2a7c2b6b521cf2baeb6652c229c4 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 13:59:04 -0800 Subject: [PATCH 032/100] move types up to connectorBuilder/ level --- .../src/components/connectorBuilder/Builder/Builder.tsx | 2 +- .../src/components/connectorBuilder/Builder/BuilderSidebar.tsx | 2 +- .../src/components/connectorBuilder/YamlEditor/YamlEditor.tsx | 2 +- .../src/components/connectorBuilder/{Builder => }/types.ts | 0 .../services/connectorBuilder/ConnectorBuilderStateService.tsx | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename airbyte-webapp/src/components/connectorBuilder/{Builder => }/types.ts (100%) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index a60d4c6c495a..15dd270a6a96 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -3,11 +3,11 @@ import { useEffect, useState } from "react"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { BuilderFormValues } from "../types"; import styles from "./Builder.module.scss"; import { BuilderSidebar, BuilderView } from "./BuilderSidebar"; import { GlobalConfigView } from "./GlobalConfigView"; import { StreamConfigView } from "./StreamConfigView"; -import { BuilderFormValues } from "./types"; interface BuilderProps { values: BuilderFormValues; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 57411a780151..f5e8971ef3f7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -14,10 +14,10 @@ import { useConnectorBuilderState, } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { BuilderFormValues } from "../types"; import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; -import { BuilderFormValues } from "./types"; import { UiYamlToggleButton } from "./UiYamlToggleButton"; export type BuilderView = "global" | number; diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 9f7721b409c6..f4500fdd5f50 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -10,8 +10,8 @@ import { ConnectorManifest } from "core/request/ConnectorBuilderClient"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; -import { convertToManifest } from "../Builder/types"; import { UiYamlToggleButton } from "../Builder/UiYamlToggleButton"; +import { convertToManifest } from "../types"; import { DownloadYamlButton } from "./DownloadYamlButton"; import styles from "./YamlEditor.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts similarity index 100% rename from airbyte-webapp/src/components/connectorBuilder/Builder/types.ts rename to airbyte-webapp/src/components/connectorBuilder/types.ts diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b0696ad8cb05..e90ce2ebb7f1 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -3,7 +3,7 @@ import React, { useContext, useEffect, useMemo, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; -import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/Builder/types"; +import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; import { ConnectorManifest, From 24eaaffce7e899ce902bbcdda3b70ff4bd15fa00 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 14:21:43 -0800 Subject: [PATCH 033/100] use grid display for ui yaml toggle button --- .../Builder/BuilderSidebar.module.scss | 2 +- .../Builder/UiYamlToggleButton.module.scss | 12 +++++++----- .../connectorBuilder/Builder/UiYamlToggleButton.tsx | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index f5b9ab771c84..dbe68eeb9e12 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -16,7 +16,7 @@ width: 90px; border-radius: 25px; padding: variables.$spacing-xl; - margin-top: 50px; + margin-top: 55px; } .connectorName { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss index c8e98efcf62c..53e26ff34d47 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss @@ -5,15 +5,16 @@ cursor: pointer; border: variables.$border-thin solid colors.$dark-blue; background-color: colors.$dark-blue; - display: flex; - border-radius: 6px; + border-radius: variables.$border-radius-sm; padding: 0; overflow: hidden; - height: 21px; - width: 96px; + + // absolute positioning so it is in the same spot in both the ui and yaml views position: absolute; top: 15px; - left: 52px; + left: 51px; + display: grid; + grid-template: 1fr / 1fr 1fr; } .text { @@ -23,6 +24,7 @@ align-items: center; justify-content: center; font-weight: 700; + padding: 4px 8px; } .selected { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx index 9e970847eb3b..8e7f636870da 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx @@ -14,7 +14,7 @@ export const UiYamlToggleButton: React.FC = ({ classNam return ( ); diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 64b0e014547c..0643b5738750 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -596,7 +596,11 @@ "connectorBuilder.testConnector": "TEST YOUR CONNECTOR", "connectorBuilder.couldNotDetectStreams": "Could not detect streams in the YAML editor:", "connectorBuilder.ensureProperYaml": "In order to test a stream, ensure that the YAML is structured as described in the docs.", - "connectorBuilder.newStream": "New stream", + "connectorBuilder.addStreamModal.title": "New stream", + "connectorBuilder.addStreamModal.streamNameLabel": "Stream name", + "connectorBuilder.addStreamModal.streamNameTooltip": "Name of the new stream", + "connectorBuilder.addStreamModal.urlPathLabel": "URL path", + "connectorBuilder.addStreamModal.urlPathTooltip": "URL path of the endpoint for this stream", "connectorBuilder.resetModal.text": "This will erase all streams and values that are currently set. Are you sure you want to continue?", "connectorBuilder.resetModal.title": "Reset Connector Builder", "connectorBuilder.resetModal.submitButton": "Reset", @@ -606,6 +610,9 @@ "connectorBuilder.toggleModal.text": "Toggling back to the UI will erase any changes you have made in the YAML editor.\n\nIn order to export your current yaml, click the Download YAML button.", "connectorBuilder.toggleModal.title": "Warning", "connectorBuilder.toggleModal.submitButton": "Confirm", + "connectorBuilder.connectorImgAlt": "Connector Image", + "connectorBuilder.uiYamlToggle.ui": "UI", + "connectorBuilder.uiYamlToggle.yaml": "YAML", "cloudApi.loginCallbackUrlError": "There was an error connecting to the developer portal. Please try again." } From b7c08f6685b523a2551e2fd8e11684f1bd472b8e Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 8 Dec 2022 15:18:30 -0800 Subject: [PATCH 036/100] pull connector manifest schema in through separate openapi spec --- airbyte-webapp/.gitignore | 1 + airbyte-webapp/orval.config.ts | 17 +++++++++++++++++ .../connectorBuilder/YamlEditor/YamlEditor.tsx | 4 ++-- .../src/components/connectorBuilder/types.ts | 2 +- .../ConnectorBuilderStateService.tsx | 15 ++++++--------- .../connector_manifest_openapi.yaml | 9 +++++++++ 6 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml diff --git a/airbyte-webapp/.gitignore b/airbyte-webapp/.gitignore index 8e572d98ba49..4c2aede53775 100644 --- a/airbyte-webapp/.gitignore +++ b/airbyte-webapp/.gitignore @@ -32,3 +32,4 @@ storybook-static/ # Ignore generated API clients, since they're automatically generated /src/core/request/AirbyteClient.ts /src/core/request/ConnectorBuilderClient.ts +/src/core/request/ConnectorManifest.ts diff --git a/airbyte-webapp/orval.config.ts b/airbyte-webapp/orval.config.ts index f859eaa5e4e2..f32b8e5fa769 100644 --- a/airbyte-webapp/orval.config.ts +++ b/airbyte-webapp/orval.config.ts @@ -43,4 +43,21 @@ export default defineConfig({ }, }, }, + connectorManifest: { + input: "./src/services/connectorBuilder/connector_manifest_openapi.yaml", + output: { + target: "./src/core/request/ConnectorManifest.ts", + prettier: true, + override: { + header: (info) => [ + `eslint-disable`, + `Generated by orval 🍺`, + `Do not edit manually. Run "npm run generate-client" instead.`, + ...(info.title ? [info.title] : []), + ...(info.description ? [info.description] : []), + ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []), + ], + }, + }, + }, }); diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 0e2ca59e9b66..3c6d4b7d2516 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -6,7 +6,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { CodeEditor } from "components/ui/CodeEditor"; -import { StreamsListRequestBodyManifest } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; @@ -41,7 +41,7 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { const yamlEditorModel = yamlEditorRef.current.getModel(); try { - const json = load(yamlValue) as StreamsListRequestBodyManifest; + const json = load(yamlValue) as ConnectorManifest; setJsonManifest(json); setYamlIsValid(true); diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index b95825de8c95..5917ec348ea4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -1,4 +1,4 @@ -import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorManifest"; export interface BuilderFormValues { global: { diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 7c21bba76da3..c1b3a94a6a96 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -5,11 +5,8 @@ import { useLocalStorage } from "react-use"; import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; -import { - StreamReadRequestBodyConfig, - StreamReadRequestBodyManifest, - StreamsListReadStreamsItem, -} from "core/request/ConnectorBuilderClient"; +import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; +import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useListStreams } from "./ConnectorBuilderApiService"; @@ -22,7 +19,7 @@ export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { streams: [], }; -const DEFAULT_JSON_MANIFEST_VALUES: StreamReadRequestBodyManifest = { +const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { version: "0.1.0", check: { stream_names: [], @@ -32,7 +29,7 @@ const DEFAULT_JSON_MANIFEST_VALUES: StreamReadRequestBodyManifest = { interface Context { builderFormValues: BuilderFormValues; - jsonManifest: StreamReadRequestBodyManifest; + jsonManifest: ConnectorManifest; yamlManifest: string; yamlEditorIsMounted: boolean; yamlIsValid: boolean; @@ -42,7 +39,7 @@ interface Context { configString: string; configJson: StreamReadRequestBodyConfig; setBuilderFormValues: (values: BuilderFormValues) => void; - setJsonManifest: (jsonValue: StreamReadRequestBodyManifest) => void; + setJsonManifest: (jsonValue: ConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setSelectedStream: (streamName: string) => void; @@ -61,7 +58,7 @@ export const ConnectorBuilderStateProvider: React.FC( + const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", DEFAULT_JSON_MANIFEST_VALUES ); diff --git a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml new file mode 100644 index 000000000000..89fe28623293 --- /dev/null +++ b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml @@ -0,0 +1,9 @@ +openapi: 3.0.0 +info: + title: Connector Manifest schema + version: 1.0.0 +paths: {} +components: + schemas: + ConnectorManifest: + $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json" From d6f10b4838ecacafaa3fb1c29eeff57b252e26c9 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 14:43:03 -0800 Subject: [PATCH 037/100] use correct intl string id --- .../src/components/connectorBuilder/Builder/AddStreamButton.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 76ed0101af99..26fb54e12611 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -59,7 +59,7 @@ export const AddStreamButton: React.FC = ({ className, onA } + title={} onClose={() => { setIsOpen(false); }} From 5ffe20e6702cf36bf629ec9acaa5ae60168761fb Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 15:15:53 -0800 Subject: [PATCH 038/100] throttle setting json manifest in yaml editor --- .../components/connectorBuilder/YamlEditor/YamlEditor.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx index 3c6d4b7d2516..53e940448f20 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/YamlEditor/YamlEditor.tsx @@ -1,5 +1,6 @@ import { useMonaco } from "@monaco-editor/react"; import { load, YAMLException } from "js-yaml"; +import debounce from "lodash/debounce"; import isMatch from "lodash/isMatch"; import { editor } from "monaco-editor/esm/vs/editor/editor.api"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -33,6 +34,9 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { } = useConnectorBuilderState(); const [yamlValue, setYamlValue] = useState(yamlManifest); + // debounce the setJsonManifest calls so that it doesnt result in a network call for every keystroke + const debouncedSetJsonManifest = useMemo(() => debounce(setJsonManifest, 200), [setJsonManifest]); + const monaco = useMonaco(); useEffect(() => { @@ -42,8 +46,8 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { try { const json = load(yamlValue) as ConnectorManifest; - setJsonManifest(json); setYamlIsValid(true); + debouncedSetJsonManifest(json); // clear editor error markers if (yamlEditorModel) { @@ -70,7 +74,7 @@ export const YamlEditor: React.FC = ({ toggleYamlEditor }) => { } } } - }, [yamlValue, monaco, setJsonManifest, setYamlIsValid]); + }, [yamlValue, monaco, debouncedSetJsonManifest, setYamlIsValid]); const yamlIsDirty = useMemo(() => { return !isMatch(convertToManifest(builderFormValues), jsonManifest); From b82ba2af178955b56dbb43cfd8ca30a9f15f20a1 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 15:21:04 -0800 Subject: [PATCH 039/100] use button prop instead of manually styling --- .../StreamTestingPanel/StreamTester.module.scss | 4 ---- .../connectorBuilder/StreamTestingPanel/StreamTester.tsx | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss index 0d538a3b55b8..eaee7846fc86 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss @@ -18,10 +18,6 @@ z-index: 0; } -.testButton { - width: 100%; -} - .testButtonTooltipContainer { width: 100%; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 3cb1cfa7e4a1..4c4cc58f42cc 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -60,7 +60,7 @@ export const StreamTester: React.FC = ({ selectedStream }) => const testButton = (
From 15872432b0cd03a5dd625853076a147694835dfe Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 15:46:15 -0800 Subject: [PATCH 041/100] fix sidebar flex styles --- .../connectorBuilder/Builder/BuilderSidebar.module.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 8cd5caabde7d..1729fd32bb36 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -47,7 +47,6 @@ } .downloadButton { - margin-top: auto; width: 90%; } @@ -60,7 +59,7 @@ .viewButton { border: none; width: 100%; - height: 34px; + flex: 0 0 34px; cursor: pointer; border-radius: variables.$border-radius-md; padding: variables.$spacing-md; @@ -89,6 +88,7 @@ } .streamList { + flex: 1; width: 100%; margin: variables.$spacing-md 0; display: flex; From 62a60e5ca7576f5997e05f1c5a5910666b18fd94 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 15:48:02 -0800 Subject: [PATCH 042/100] use specific flex properties instead of flex --- .../connectorBuilder/Builder/BuilderSidebar.module.scss | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 1729fd32bb36..9d1e68667916 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -43,7 +43,7 @@ .streamsHeading { color: colors.$blue; - flex: 1; + flex-grow: 1; } .downloadButton { @@ -59,7 +59,8 @@ .viewButton { border: none; width: 100%; - flex: 0 0 34px; + height: 34px; + flex-shrink: 0; cursor: pointer; border-radius: variables.$border-radius-md; padding: variables.$spacing-md; @@ -88,7 +89,7 @@ } .streamList { - flex: 1; + flex-grow: 1; width: 100%; margin: variables.$spacing-md 0; display: flex; From 092821dca2a898ef015e19ee8f1c5b153683e3d6 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 9 Dec 2022 16:12:35 -0800 Subject: [PATCH 043/100] clean up download and reset button styles --- .../Builder/BuilderSidebar.module.scss | 4 +--- .../connectorBuilder/Builder/BuilderSidebar.tsx | 13 ++++--------- .../{YamlEditor => }/DownloadYamlButton.tsx | 4 +--- .../YamlEditor/DownloadYamlButton.module.scss | 4 ---- .../connectorBuilder/YamlEditor/YamlEditor.tsx | 2 +- airbyte-webapp/src/locales/en.json | 1 + 6 files changed, 8 insertions(+), 20 deletions(-) rename airbyte-webapp/src/components/connectorBuilder/{YamlEditor => }/DownloadYamlButton.tsx (93%) delete mode 100644 airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.module.scss diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index 9d1e68667916..36a6f949e28a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -47,13 +47,11 @@ } .downloadButton { - width: 90%; + width: 100%; } .resetButton { margin-top: variables.$spacing-sm; - margin-bottom: variables.$spacing-md; - width: 90%; } .viewButton { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index f337e20145de..1e497c264ab7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,4 +1,4 @@ -import { faRotateLeft, faSliders } from "@fortawesome/free-solid-svg-icons"; +import { faSliders } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useFormikContext } from "formik"; @@ -14,8 +14,8 @@ import { useConnectorBuilderState, } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { DownloadYamlButton } from "../DownloadYamlButton"; import { BuilderFormValues } from "../types"; -import { DownloadYamlButton } from "../YamlEditor/DownloadYamlButton"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; import { UiYamlToggleButton } from "./UiYamlToggleButton"; @@ -122,13 +122,8 @@ export const BuilderSidebar: React.FC = ({
-
); diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx similarity index 93% rename from airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx rename to airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx index 1efbd049346a..a807239e778f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx @@ -7,8 +7,6 @@ import { Tooltip } from "components/ui/Tooltip"; import { downloadFile } from "utils/file"; -import styles from "./DownloadYamlButton.module.scss"; - interface DownloadYamlButtonProps { className?: string; yaml: string; @@ -24,7 +22,7 @@ export const DownloadYamlButton: React.FC = ({ classNam const downloadButton = ( - + {!renderInputForm && ( + + + + )} )} diff --git a/airbyte-webapp/src/core/domain/connector/connector.ts b/airbyte-webapp/src/core/domain/connector/connector.ts index 6f0749c43952..c9dfc117f285 100644 --- a/airbyte-webapp/src/core/domain/connector/connector.ts +++ b/airbyte-webapp/src/core/domain/connector/connector.ts @@ -1,5 +1,10 @@ import { DEV_IMAGE_TAG } from "./constants"; -import { isSource, isSourceDefinition, isSourceDefinitionSpecification } from "./source"; +import { + isSource, + isSourceDefinition, + isSourceDefinitionSpecification, + isSourceDefinitionSpecificationDraft, +} from "./source"; import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT } from "./types"; export class Connector { @@ -27,6 +32,9 @@ export class ConnectorHelper { export class ConnectorSpecification { static id(connector: ConnectorDefinitionSpecification): string { + if (isSourceDefinitionSpecificationDraft(connector)) { + throw new Error("Tried to get id for specification draft"); + } return isSourceDefinitionSpecification(connector) ? connector.sourceDefinitionId : connector.destinationDefinitionId; diff --git a/airbyte-webapp/src/core/domain/connector/source.ts b/airbyte-webapp/src/core/domain/connector/source.ts index 9ecb47a49962..d00c45e8a8f6 100644 --- a/airbyte-webapp/src/core/domain/connector/source.ts +++ b/airbyte-webapp/src/core/domain/connector/source.ts @@ -1,5 +1,15 @@ -import { SourceDefinitionRead, SourceDefinitionSpecificationRead, SourceRead } from "../../request/AirbyteClient"; -import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT } from "./types"; +import { + DestinationDefinitionSpecificationRead, + SourceDefinitionRead, + SourceDefinitionSpecificationRead, + SourceRead, +} from "../../request/AirbyteClient"; +import { + ConnectorDefinition, + ConnectorDefinitionSpecification, + ConnectorT, + SourceDefinitionSpecificationDraft, +} from "./types"; export function isSource(connector: ConnectorT): connector is SourceRead { return "sourceId" in connector; @@ -15,5 +25,14 @@ export function isSourceDefinitionSpecification( return (connector as SourceDefinitionSpecificationRead).sourceDefinitionId !== undefined; } +export function isSourceDefinitionSpecificationDraft( + connector: ConnectorDefinitionSpecification +): connector is SourceDefinitionSpecificationDraft { + return ( + (connector as SourceDefinitionSpecificationRead).sourceDefinitionId === undefined && + (connector as DestinationDefinitionSpecificationRead).destinationDefinitionId === undefined + ); +} + // eslint-disable-next-line no-template-curly-in-string export const SOURCE_NAMESPACE_TAG = "${SOURCE_NAMESPACE}"; diff --git a/airbyte-webapp/src/core/domain/connector/types.ts b/airbyte-webapp/src/core/domain/connector/types.ts index d52ec5feb3c2..87ad8bce46c2 100644 --- a/airbyte-webapp/src/core/domain/connector/types.ts +++ b/airbyte-webapp/src/core/domain/connector/types.ts @@ -10,8 +10,14 @@ import { export type ConnectorDefinition = SourceDefinitionReadWithLatestTag | DestinationDefinitionReadWithLatestTag; +export type SourceDefinitionSpecificationDraft = Pick< + SourceDefinitionSpecificationRead, + "documentationUrl" | "connectionSpecification" | "authSpecification" | "advancedAuth" +>; + export type ConnectorDefinitionSpecification = | DestinationDefinitionSpecificationRead - | SourceDefinitionSpecificationRead; + | SourceDefinitionSpecificationRead + | SourceDefinitionSpecificationDraft; export type ConnectorT = DestinationRead | SourceRead; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index af36bd17e642..40451d4a3abe 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -186,6 +186,7 @@ "connectorForm.authenticate.required": "Authentication required", "connectorForm.reauthenticate": "Re-authenticate", "connectorForm.expandForm": "Expand this form to continue setting up your connector", + "connectorForm.saveInputsForm": "Save", "form.rawData": "Raw data (JSON)", "form.basicNormalization": "Normalized tabular data", diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index 9ba86f33bf11..7eacc4d5686e 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -16,7 +16,10 @@ import { useBuildForm, useConstructValidationSchema } from "./useBuildForm"; export interface ConnectorFormProps { formType: "source" | "destination"; formId?: string; - selectedConnectorDefinition: ConnectorDefinition; + /** + * Definition of the connector might not be available if it's not released but only exists in frontend heap + */ + selectedConnectorDefinition?: ConnectorDefinition; selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; onSubmit: (values: ConnectorFormValues) => Promise; isEditMode?: boolean; @@ -25,6 +28,12 @@ export interface ConnectorFormProps { errorMessage?: React.ReactNode; successMessage?: React.ReactNode; connectorId?: string; + footerClassName?: string; + submitLabel?: string; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; isTestConnectionInProgress?: boolean; onStopTesting?: () => void; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx index b88b7b6fa556..991ca5a7cf7f 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx @@ -17,6 +17,12 @@ interface FormRootProps { successMessage?: React.ReactNode; onRetest?: () => void; onStopTestingConnector?: () => void; + submitLabel?: string; + footerClassName?: string; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; } export const FormRoot: React.FC = ({ @@ -27,6 +33,9 @@ export const FormRoot: React.FC = ({ errorMessage, hasSuccess, onStopTestingConnector, + submitLabel, + footerClassName, + onCancel, }) => { const { dirty, isSubmitting, isValid } = useFormikContext(); const { resetConnectorForm, isEditMode, formType } = useConnectorForm(); @@ -34,31 +43,35 @@ export const FormRoot: React.FC = ({ return (
- {isEditMode ? ( - { - resetConnectorForm(); - }} - successMessage={successMessage} - /> - ) : ( - - )} +
+ {isEditMode ? ( + { + resetConnectorForm(); + }} + successMessage={successMessage} + /> + ) : ( + + )} +
); }; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss index e6049bbfa295..8b96a3953685 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.module.scss @@ -1,3 +1,15 @@ -.submitButton { - margin-left: auto; +@use "scss/variables"; + +.controlContainer { + margin-top: 34px; + display: flex; + align-items: center; + justify-content: flex-end; +} + +.buttonContainer { + display: flex; + flex: 0 0 auto; + align-self: flex-end; + gap: variables.$spacing-sm; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx index 83ec70e28db2..5728dab65f8c 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/CreateControls.tsx @@ -1,6 +1,5 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; import { Button } from "components/ui/Button"; @@ -11,6 +10,11 @@ import TestingConnectionSuccess from "./TestingConnectionSuccess"; interface CreateControlProps { formType: "source" | "destination"; + /** + * Called in case the user cancels the form - if not provided, no cancel button is rendered + */ + onCancel?: () => void; + submitLabel?: string; isSubmitting: boolean; errorMessage?: React.ReactNode; hasSuccess?: boolean; @@ -19,13 +23,6 @@ interface CreateControlProps { onCancelTesting?: () => void; } -const ButtonContainer = styled.div` - margin-top: 34px; - display: flex; - align-items: center; - justify-content: space-between; -`; - const CreateControls: React.FC = ({ isTestConnectionInProgress, isSubmitting, @@ -33,6 +30,8 @@ const CreateControls: React.FC = ({ hasSuccess, errorMessage, onCancelTesting, + onCancel, + submitLabel, }) => { if (isSubmitting) { return ; @@ -43,12 +42,19 @@ const CreateControls: React.FC = ({ } return ( - +
{errorMessage && } - - +
+ {onCancel && ( + + )} + +
+
); }; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx index ed93c3bff8d8..3e6dae24af56 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx @@ -54,7 +54,7 @@ export const AuthButton: React.FC = () => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const { loading, done, run, hasRun } = useFormikOauthAdapter(selectedConnectorDefinitionSpecification); - if (!selectedConnectorDefinitionSpecification) { + if (!selectedConnectorDefinitionSpecification || !selectedConnectorDefinition) { console.error("Entered non-auth flow while no connector is selected"); return null; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx index 11d2f766536f..7d4d10e7daa1 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx @@ -10,7 +10,7 @@ interface ConnectorFormContext { formType: "source" | "destination"; getValues: (values: ConnectorFormValues) => ConnectorFormValues; resetConnectorForm: () => void; - selectedConnectorDefinition: ConnectorDefinition; + selectedConnectorDefinition?: ConnectorDefinition; selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; isEditMode?: boolean; validationSchema: AnySchema; @@ -28,7 +28,7 @@ export const useConnectorForm = (): ConnectorFormContext => { }; interface ConnectorFormContextProviderProps { - selectedConnectorDefinition: ConnectorDefinition; + selectedConnectorDefinition?: ConnectorDefinition; formType: "source" | "destination"; isEditMode?: boolean; getValues: (values: ConnectorFormValues) => ConnectorFormValues; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index 3c561f041dc2..8e3246c1d32d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -4,6 +4,7 @@ import { useIntl } from "react-intl"; import { AnySchema } from "yup"; import { ConnectorDefinitionSpecification } from "core/domain/connector"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { FormBuildError } from "core/form/FormBuildError"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; @@ -25,22 +26,29 @@ export function useBuildForm( ): BuildFormHook { const { formatMessage } = useIntl(); - const jsonSchema: JSONSchema7 = useMemo( - () => ({ + const isDraft = isSourceDefinitionSpecificationDraft(selectedConnectorDefinitionSpecification); + + const jsonSchema: JSONSchema7 = useMemo(() => { + const schema: JSONSchema7 = { type: "object", properties: { - name: { - type: "string", - title: formatMessage({ id: `form.${formType}Name` }), - description: formatMessage({ id: `form.${formType}Name.message` }), - }, connectionConfiguration: selectedConnectorDefinitionSpecification.connectionSpecification as JSONSchema7Definition, }, - required: ["name"], - }), - [formType, formatMessage, selectedConnectorDefinitionSpecification.connectionSpecification] - ); + }; + if (isDraft) { + return schema; + } + // schema.properties gets defined right above + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + schema.properties!.name = { + type: "string", + title: formatMessage({ id: `form.${formType}Name` }), + description: formatMessage({ id: `form.${formType}Name.message` }), + }; + schema.required = ["name"]; + return schema; + }, [formType, formatMessage, isDraft, selectedConnectorDefinitionSpecification.connectionSpecification]); const formFields = useMemo(() => jsonSchemaToFormBlock(jsonSchema), [jsonSchema]); @@ -49,7 +57,7 @@ export function useBuildForm( } const startValues = useMemo(() => { - if (isEditMode) { + if (isEditMode || isDraft) { return { name: "", connectionConfiguration: {}, @@ -85,7 +93,7 @@ export function useBuildForm( setDefaultValues(formFields, baseValues as Record); return baseValues; - }, [formFields, initialValues, isEditMode]); + }, [formFields, initialValues, isDraft, isEditMode]); return { initialValues: startValues, From 901fc634a98e33c90042b94da05362f076f2e53c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 12 Dec 2022 20:44:38 +0100 Subject: [PATCH 053/100] wip --- .../connectorBuilder/Builder/BuilderSidebar.tsx | 13 +++++++++++-- airbyte-webapp/src/locales/en.json | 1 + .../ConnectorBuilderStateService.tsx | 6 +++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 2a2b40827cb0..bbf8dc1f3b59 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,4 +1,4 @@ -import { faSliders } from "@fortawesome/free-solid-svg-icons"; +import { faSliders, faPerson } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useFormikContext } from "formik"; @@ -70,7 +70,7 @@ export const BuilderSidebar: React.FC = ({ className, toggl }; const handleViewSelect = (selectedView: BuilderView) => { setSelectedView(selectedView); - if (selectedView !== "global") { + if (selectedView !== "global" && selectedView !== "inputs") { setTestStreamIndex(selectedView); } }; @@ -101,6 +101,15 @@ export const BuilderSidebar: React.FC = ({ className, toggl + handleViewSelect("inputs")} + > + + + +
diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 40451d4a3abe..a85900504644 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -650,6 +650,7 @@ "connectorBuilder.resetModal.submitButton": "Reset", "connectorBuilder.streamsHeading": "STREAMS ({number})", "connectorBuilder.globalConfiguration": "Global Configuration", + "connectorBuilder.userInputs": "User inputs ({number})", "connectorBuilder.noStreamsMessage": "Add a stream to test it here", "connectorBuilder.toggleModal.text": "Toggling back to the UI will erase any changes you have made in the YAML editor.\n\nIn order to export your current yaml, click the Download YAML button.", "connectorBuilder.toggleModal.title": "Warning", diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 3e079cb14a07..553c0ad3d34c 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -26,7 +26,7 @@ const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { streams: [], }; -export type BuilderView = "global" | number; +export type BuilderView = "global" | "inputs" | number; interface Context { builderFormValues: BuilderFormValues; @@ -67,6 +67,10 @@ export const ConnectorBuilderStateProvider: React.FC { + // jsonManifest. + // }, []); + useEffect(() => { setJsonManifest(convertToManifest(formValues)); }, [formValues, setJsonManifest]); From 3255298b55c03e0aa7e00085343767037d318327 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 13 Dec 2022 10:43:54 +0100 Subject: [PATCH 054/100] fix small stuff --- .../src/core/domain/connector/connector.ts | 16 ++++------ .../src/core/domain/connector/source.ts | 2 +- .../src/core/domain/connector/types.ts | 3 +- .../PatchedConnectorManifest.ts | 8 +++++ .../ConnectorBuilderStateService.tsx | 3 +- .../Connector/ConnectorForm/ConnectorForm.tsx | 8 +++-- .../Sections/auth/AuthButton.test.tsx | 30 +++++++++++++------ .../components/Sections/auth/AuthButton.tsx | 12 ++++---- .../components/Sections/auth/AuthSection.tsx | 8 ++++- .../ConnectorForm/connectorFormContext.tsx | 10 +++++-- .../Connector/ConnectorForm/index.stories.tsx | 12 ++++---- .../ConnectorForm/useAuthentication.tsx | 8 +++-- .../Connector/ConnectorForm/useBuildForm.tsx | 4 +-- 13 files changed, 78 insertions(+), 46 deletions(-) create mode 100644 airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts diff --git a/airbyte-webapp/src/core/domain/connector/connector.ts b/airbyte-webapp/src/core/domain/connector/connector.ts index c9dfc117f285..d9452f34c211 100644 --- a/airbyte-webapp/src/core/domain/connector/connector.ts +++ b/airbyte-webapp/src/core/domain/connector/connector.ts @@ -1,11 +1,8 @@ +import { DestinationDefinitionSpecificationRead, SourceDefinitionSpecificationRead } from "core/request/AirbyteClient"; + import { DEV_IMAGE_TAG } from "./constants"; -import { - isSource, - isSourceDefinition, - isSourceDefinitionSpecification, - isSourceDefinitionSpecificationDraft, -} from "./source"; -import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT } from "./types"; +import { isSource, isSourceDefinition, isSourceDefinitionSpecification } from "./source"; +import { ConnectorDefinition, ConnectorT } from "./types"; export class Connector { static id(connector: ConnectorDefinition): string { @@ -31,10 +28,7 @@ export class ConnectorHelper { } export class ConnectorSpecification { - static id(connector: ConnectorDefinitionSpecification): string { - if (isSourceDefinitionSpecificationDraft(connector)) { - throw new Error("Tried to get id for specification draft"); - } + static id(connector: DestinationDefinitionSpecificationRead | SourceDefinitionSpecificationRead): string { return isSourceDefinitionSpecification(connector) ? connector.sourceDefinitionId : connector.destinationDefinitionId; diff --git a/airbyte-webapp/src/core/domain/connector/source.ts b/airbyte-webapp/src/core/domain/connector/source.ts index d00c45e8a8f6..ae43b1df16e4 100644 --- a/airbyte-webapp/src/core/domain/connector/source.ts +++ b/airbyte-webapp/src/core/domain/connector/source.ts @@ -26,7 +26,7 @@ export function isSourceDefinitionSpecification( } export function isSourceDefinitionSpecificationDraft( - connector: ConnectorDefinitionSpecification + connector: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft ): connector is SourceDefinitionSpecificationDraft { return ( (connector as SourceDefinitionSpecificationRead).sourceDefinitionId === undefined && diff --git a/airbyte-webapp/src/core/domain/connector/types.ts b/airbyte-webapp/src/core/domain/connector/types.ts index 87ad8bce46c2..444af7d0e98b 100644 --- a/airbyte-webapp/src/core/domain/connector/types.ts +++ b/airbyte-webapp/src/core/domain/connector/types.ts @@ -17,7 +17,6 @@ export type SourceDefinitionSpecificationDraft = Pick< export type ConnectorDefinitionSpecification = | DestinationDefinitionSpecificationRead - | SourceDefinitionSpecificationRead - | SourceDefinitionSpecificationDraft; + | SourceDefinitionSpecificationRead; export type ConnectorT = DestinationRead | SourceRead; diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts b/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts new file mode 100644 index 000000000000..2e8dd79b575f --- /dev/null +++ b/airbyte-webapp/src/core/domain/connectorBuilder/PatchedConnectorManifest.ts @@ -0,0 +1,8 @@ +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; + +import { ConnectorManifest } from "../../request/ConnectorManifest"; + +// Patching this type as required until the upstream schema is updated +export interface PatchedConnectorManifest extends ConnectorManifest { + spec?: SourceDefinitionSpecificationDraft; +} diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 3e079cb14a07..b83d73969ff6 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -5,6 +5,7 @@ import { useLocalStorage } from "react-use"; import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; +import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { ConnectorManifest } from "core/request/ConnectorManifest"; @@ -30,7 +31,7 @@ export type BuilderView = "global" | number; interface Context { builderFormValues: BuilderFormValues; - jsonManifest: ConnectorManifest; + jsonManifest: PatchedConnectorManifest; yamlManifest: string; yamlEditorIsMounted: boolean; yamlIsValid: boolean; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index 7eacc4d5686e..8a292ae952b4 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -3,7 +3,11 @@ import React, { useCallback } from "react"; import { FormChangeTracker } from "components/common/FormChangeTracker"; -import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; +import { + ConnectorDefinition, + ConnectorDefinitionSpecification, + SourceDefinitionSpecificationDraft, +} from "core/domain/connector"; import { FormikPatch } from "core/form/FormikPatch"; import { CheckConnectionRead } from "core/request/AirbyteClient"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; @@ -20,7 +24,7 @@ export interface ConnectorFormProps { * Definition of the connector might not be available if it's not released but only exists in frontend heap */ selectedConnectorDefinition?: ConnectorDefinition; - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft; onSubmit: (values: ConnectorFormValues) => Promise; isEditMode?: boolean; formValues?: Partial; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx index 7ad988453e15..0d35ea01ee9d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.test.tsx @@ -55,9 +55,9 @@ describe("auth button", () => { it("initially renders with correct message and no status message", () => { // no auth errors mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // not done @@ -70,7 +70,11 @@ describe("auth button", () => { render( - + ); @@ -90,9 +94,9 @@ describe("auth button", () => { it("after successful authentication, it renders with correct message and success message", () => { // no auth errors mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // done @@ -105,7 +109,11 @@ describe("auth button", () => { render( - + ); @@ -123,9 +131,9 @@ describe("auth button", () => { mockUseAuthentication.mockReturnValue({ hiddenAuthFieldErrors: { field: "form.empty.error" } }); mockUseConnectorForm.mockImplementationOnce(() => { - const { selectedConnectorDefinitionSpecification, selectedConnectorDefinition } = baseUseConnectorFormValues; + const { selectedConnectorDefinition } = baseUseConnectorFormValues; - return { selectedConnectorDefinitionSpecification, selectedConnectorDefinition }; + return { selectedConnectorDefinition }; }); // not done @@ -138,7 +146,11 @@ describe("auth button", () => { render( - + ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx index 3e6dae24af56..df7b33ab31e5 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthButton.tsx @@ -5,7 +5,7 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Text } from "components/ui/Text"; -import { ConnectorSpecification } from "core/domain/connector"; +import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; import { ConnectorIds } from "utils/connectors"; import { useConnectorForm } from "../../../connectorFormContext"; @@ -46,16 +46,18 @@ function getAuthenticateMessageId(connectorDefinitionId: string): string { return "connectorForm.authenticate"; } -export const AuthButton: React.FC = () => { - const { selectedConnectorDefinition, selectedConnectorDefinitionSpecification } = useConnectorForm(); +export const AuthButton: React.FC<{ + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; +}> = ({ selectedConnectorDefinitionSpecification }) => { + const { selectedConnectorDefinition } = useConnectorForm(); const { hiddenAuthFieldErrors } = useAuthentication(); const authRequiredError = Object.values(hiddenAuthFieldErrors).includes("form.empty.error"); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const { loading, done, run, hasRun } = useFormikOauthAdapter(selectedConnectorDefinitionSpecification); - if (!selectedConnectorDefinitionSpecification || !selectedConnectorDefinition) { - console.error("Entered non-auth flow while no connector is selected"); + if (!selectedConnectorDefinition) { + console.error("Entered non-auth flow while no supported connector is selected"); return null; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx index 1c89cf2b1e62..12b38123404d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx @@ -1,15 +1,21 @@ import React from "react"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { FeatureItem, IfFeatureEnabled } from "hooks/services/Feature"; +import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormContext"; import { SectionContainer } from "../SectionContainer"; import { AuthButton } from "./AuthButton"; export const AuthSection: React.FC = () => { + const { selectedConnectorDefinitionSpecification } = useConnectorForm(); + if (isSourceDefinitionSpecificationDraft(selectedConnectorDefinitionSpecification)) { + return null; + } return ( - + ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx index 7d4d10e7daa1..a222e14c59c9 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx @@ -2,7 +2,11 @@ import { useFormikContext } from "formik"; import React, { useContext, useMemo } from "react"; import { AnySchema } from "yup"; -import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; +import { + ConnectorDefinition, + ConnectorDefinitionSpecification, + SourceDefinitionSpecificationDraft, +} from "core/domain/connector"; import { ConnectorFormValues } from "./types"; @@ -11,7 +15,7 @@ interface ConnectorFormContext { getValues: (values: ConnectorFormValues) => ConnectorFormValues; resetConnectorForm: () => void; selectedConnectorDefinition?: ConnectorDefinition; - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft; isEditMode?: boolean; validationSchema: AnySchema; connectorId?: string; @@ -32,7 +36,7 @@ interface ConnectorFormContextProviderProps { formType: "source" | "destination"; isEditMode?: boolean; getValues: (values: ConnectorFormValues) => ConnectorFormValues; - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft; validationSchema: AnySchema; connectorId?: string; } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx index 9ca0171a17b6..d391f35ee0a2 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/index.stories.tsx @@ -3,7 +3,7 @@ import withMock from "storybook-addon-mock"; import { Card } from "components/ui/Card"; -import { ConnectorSpecification } from "core/domain/connector"; +import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; import { isSourceDefinitionSpecification } from "core/domain/connector/source"; import { ConnectorForm } from "./ConnectorForm"; @@ -46,13 +46,11 @@ export default { } as ComponentMeta; const Template: ComponentStory = (args) => { + const selectedSpecification = args.selectedConnectorDefinitionSpecification as ConnectorDefinitionSpecification; // Hack to allow devs to not specify sourceDefinitionId - if ( - args.selectedConnectorDefinitionSpecification && - !ConnectorSpecification.id(args.selectedConnectorDefinitionSpecification) - ) { - if (isSourceDefinitionSpecification(args.selectedConnectorDefinitionSpecification)) { - args.selectedConnectorDefinitionSpecification.sourceDefinitionId = TempConnector.sourceDefinitionId; + if (!ConnectorSpecification.id(selectedSpecification)) { + if (isSourceDefinitionSpecification(selectedSpecification)) { + selectedSpecification.sourceDefinitionId = TempConnector.sourceDefinitionId; } } diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx index 7ec5a265e1b6..9610895dcf42 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.tsx @@ -3,6 +3,7 @@ import { JSONSchema7 } from "json-schema"; import { useCallback, useMemo } from "react"; import { ConnectorSpecification } from "core/domain/connector"; +import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { FeatureItem, useFeature } from "hooks/services/Feature"; @@ -155,7 +156,10 @@ export const useAuthentication = (): AuthenticationHook => { console.error(`getValues in useAuthentication failed.`, e); trackError(e, { id: "useAuthentication.getValues", - connector: connectorSpec ? ConnectorSpecification.id(connectorSpec) : null, + connector: + connectorSpec && !isSourceDefinitionSpecificationDraft(connectorSpec) + ? ConnectorSpecification.id(connectorSpec) + : null, }); return values; } @@ -178,7 +182,7 @@ export const useAuthentication = (): AuthenticationHook => { const implicitAuthFieldPaths = useMemo( () => [ // Fields from `advancedAuth` connectors - ...(advancedAuth + ...(advancedAuth && !isSourceDefinitionSpecificationDraft(connectorSpec) ? Object.values(serverProvidedOauthPaths(connectorSpec)).map((f) => makeConnectionConfigurationPath(f.path_in_connector_config) ) diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index 8e3246c1d32d..4df4eaf4c389 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -3,7 +3,7 @@ import { useMemo } from "react"; import { useIntl } from "react-intl"; import { AnySchema } from "yup"; -import { ConnectorDefinitionSpecification } from "core/domain/connector"; +import { ConnectorDefinitionSpecification, SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { isSourceDefinitionSpecificationDraft } from "core/domain/connector/source"; import { FormBuildError } from "core/form/FormBuildError"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; @@ -21,7 +21,7 @@ export interface BuildFormHook { export function useBuildForm( isEditMode: boolean, formType: "source" | "destination", - selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification, + selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification | SourceDefinitionSpecificationDraft, initialValues?: Partial ): BuildFormHook { const { formatMessage } = useIntl(); From ba280147299a8edd927e009779270b6a3db7b558 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 13 Dec 2022 14:48:21 +0100 Subject: [PATCH 055/100] add basic input UI --- .../connectorBuilder/Builder/Builder.tsx | 21 +- .../connectorBuilder/Builder/BuilderField.tsx | 30 ++- .../Builder/BuilderSidebar.tsx | 6 +- .../connectorBuilder/Builder/InputsView.tsx | 193 ++++++++++++++++++ .../src/components/connectorBuilder/types.ts | 26 ++- .../ConnectorBuilderStateService.tsx | 12 +- 6 files changed, 266 insertions(+), 22 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index fb40a24a2d9f..9d713f99fb6d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -1,12 +1,13 @@ import { Form } from "formik"; import { useEffect } from "react"; -import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; +import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { BuilderFormValues } from "../types"; import styles from "./Builder.module.scss"; import { BuilderSidebar } from "./BuilderSidebar"; import { GlobalConfigView } from "./GlobalConfigView"; +import { InputsView } from "./InputsView"; import { StreamConfigView } from "./StreamConfigView"; interface BuilderProps { @@ -14,8 +15,20 @@ interface BuilderProps { toggleYamlEditor: () => void; } +function getView(selectedView: BuilderView) { + switch (selectedView) { + case "global": + return ; + case "inputs": + return ; + default: + return ; + } +} + export const Builder: React.FC = ({ values, toggleYamlEditor }) => { - const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); + const { setBuilderFormValues, selectedView, builderFormValues } = useConnectorBuilderState(); + console.log(builderFormValues); useEffect(() => { setBuilderFormValues(values); }, [values, setBuilderFormValues]); @@ -23,9 +36,7 @@ export const Builder: React.FC = ({ values, toggleYamlEditor }) => return (
-
- {selectedView === "global" ? : } - +
{getView(selectedView)}
); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index bd799ff8efed..1677b563eda1 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -5,6 +5,7 @@ import * as yup from "yup"; import { ControlLabels } from "components/LabeledControl"; import { DropDown } from "components/ui/DropDown"; import { Input } from "components/ui/Input"; +import { Switch } from "components/ui/Switch"; import { TagInput } from "components/ui/TagInput"; import { Text } from "components/ui/Text"; @@ -30,9 +31,17 @@ interface BaseFieldProps { label: string; tooltip?: string; optional?: boolean; + additionalValidation?: (schema: yup.AnySchema) => yup.AnySchema; } -type BuilderFieldProps = BaseFieldProps & ({ type: "text" } | { type: "array" } | { type: "enum"; options: string[] }); +type BuilderFieldProps = BaseFieldProps & + ( + | { type: "text" } + | { type: "number" } + | { type: "boolean" } + | { type: "array" } + | { type: "enum"; options: string[] } + ); const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { return ( @@ -52,11 +61,21 @@ const ArrayField: React.FC = ({ name, value, setValue, error }) return setValue(value)} error={error} />; }; -export const BuilderField: React.FC = ({ path, label, tooltip, optional = false, ...props }) => { - let yupSchema = props.type === "array" ? yup.array().of(yup.string()) : yup.string(); +export const BuilderField: React.FC = ({ + path, + label, + tooltip, + optional = false, + additionalValidation, + ...props +}) => { + let yupSchema: yup.AnySchema = props.type === "array" ? yup.array().of(yup.string()) : yup.string(); if (!optional) { yupSchema = yupSchema.required("form.empty.error"); } + if (additionalValidation) { + yupSchema = additionalValidation(yupSchema); + } const fieldConfig = { name: path, validate: (value: string) => { @@ -76,7 +95,10 @@ export const BuilderField: React.FC = ({ path, label, tooltip return ( - {props.type === "text" && } + {(props.type === "text" || props.type === "number") && ( + + )} + {props.type === "boolean" && } {props.type === "array" && ( )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index bbf8dc1f3b59..5f1964ab2b87 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -1,4 +1,4 @@ -import { faSliders, faPerson } from "@fortawesome/free-solid-svg-icons"; +import { faSliders, faUser } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classnames from "classnames"; import { useFormikContext } from "formik"; @@ -106,8 +106,8 @@ export const BuilderSidebar: React.FC = ({ className, toggl selected={selectedView === "inputs"} onClick={() => handleViewSelect("inputs")} > - - + +
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx new file mode 100644 index 000000000000..10d4e3c01e5b --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -0,0 +1,193 @@ +import { Form, Formik, useField } from "formik"; +import { JSONSchema7 } from "json-schema"; +import { useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; +import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; + +import { FormikPatch } from "core/form/FormikPatch"; + +import { BuilderFormValues } from "../types"; +import { BuilderField } from "./BuilderField"; + +interface InputInEditing { + key: string; + definition: JSONSchema7; + required: boolean; + isNew?: boolean; + showDefaultValueField?: boolean; +} + +export const InputsView: React.FC = () => { + const { formatMessage } = useIntl(); + const [inputs, , helpers] = useField("inputs"); + const [inputInEditing, setInputInEditing] = useState(undefined); + return ( + <> + +
    + {inputs.value.map(({ key, definition }) => ( +
  1. + +
    2. + ))} +
+ + + {inputInEditing && ( + { + helpers.setValue( + inputInEditing.isNew + ? [...inputs.value, values] + : inputs.value.map((input) => (input.key === inputInEditing.key ? values : input)) + ); + setInputInEditing(undefined); + }} + > + {({ values }) => ( + <> + + } + onClose={() => { + setInputInEditing(undefined); + }} + > +
+ + + { + const usedKeys = inputs.value.map((input) => input.key); + if (values.isNew) { + return schema.notOneOf(usedKeys); + } + return schema.notOneOf(usedKeys.filter((key) => key !== inputInEditing.key)); + }} + label={formatMessage({ id: "connectorBuilder.inputModal.key" })} + tooltip={formatMessage({ id: "connectorBuilder.inputModal.keyTooltip" })} + /> + + + + + + {values.showDefaultValueField && ( + + )} + + + + + + + +
+ + + )} + + )} + + ); + return <>TODO: List all inputs; +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 5917ec348ea4..3546d4d65521 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -1,10 +1,19 @@ -import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorManifest"; +import { JSONSchema7 } from "json-schema"; + +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; +import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; +import { DeclarativeStream } from "core/request/ConnectorManifest"; export interface BuilderFormValues { global: { connectorName: string; urlBase: string; }; + inputs: Array<{ + key: string; + required: boolean; + definition: JSONSchema7; + }>; streams: BuilderStream[]; } @@ -15,7 +24,7 @@ export interface BuilderStream { httpMethod: "GET" | "POST"; } -export const convertToManifest = (values: BuilderFormValues): ConnectorManifest => { +export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { name: stream.name, @@ -40,11 +49,24 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest }; }); + const specSchema: JSONSchema7 = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + required: values.inputs.filter((input) => input.required).map((input) => input.key), + properties: Object.fromEntries(values.inputs.map((input) => [input.key, input.definition])), + additionalProperties: true, + }; + + const spec: SourceDefinitionSpecificationDraft = { + connectionSpecification: specSchema, + }; + return { version: "0.1.0", check: { stream_names: [], }, streams: manifestStreams, + spec, }; }; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index fa6369e0a15c..267247f89834 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -7,7 +7,6 @@ import { BuilderFormValues, convertToManifest } from "components/connectorBuilde import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; -import { ConnectorManifest } from "core/request/ConnectorManifest"; import { useListStreams } from "./ConnectorBuilderApiService"; @@ -16,10 +15,11 @@ export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { connectorName: "", urlBase: "", }, + inputs: [], streams: [], }; -const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { +const DEFAULT_JSON_MANIFEST_VALUES: PatchedConnectorManifest = { version: "0.1.0", check: { stream_names: [], @@ -42,7 +42,7 @@ interface Context { configString: string; configJson: StreamReadRequestBodyConfig; setBuilderFormValues: (values: BuilderFormValues) => void; - setJsonManifest: (jsonValue: ConnectorManifest) => void; + setJsonManifest: (jsonValue: PatchedConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; setTestStreamIndex: (streamIndex: number) => void; @@ -62,16 +62,12 @@ export const ConnectorBuilderStateProvider: React.FC( + const [jsonManifest, setJsonManifest] = useLocalStorage( "connectorBuilderJsonManifest", DEFAULT_JSON_MANIFEST_VALUES ); const manifest = jsonManifest ?? DEFAULT_JSON_MANIFEST_VALUES; - // const inputs = useMemo(() => { - // jsonManifest. - // }, []); - useEffect(() => { setJsonManifest(convertToManifest(formValues)); }, [formValues, setJsonManifest]); From b694dc557ad8488aff8156243be660073ed8f86a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 14 Dec 2022 12:05:50 +0100 Subject: [PATCH 056/100] user inputs --- .../connectorBuilder/Builder/Builder.tsx | 3 +- .../connectorBuilder/Builder/BuilderField.tsx | 18 +- .../Builder/BuilderSidebar.module.scss | 3 +- .../Builder/InputsView.module.scss | 45 +++ .../connectorBuilder/Builder/InputsView.tsx | 280 ++++++++++-------- .../StreamTestingPanel/ConfigMenu.module.scss | 4 + airbyte-webapp/src/locales/en.json | 22 ++ 7 files changed, 252 insertions(+), 123 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index 9d713f99fb6d..4107fe733158 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -27,8 +27,7 @@ function getView(selectedView: BuilderView) { } export const Builder: React.FC = ({ values, toggleYamlEditor }) => { - const { setBuilderFormValues, selectedView, builderFormValues } = useConnectorBuilderState(); - console.log(builderFormValues); + const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); useEffect(() => { setBuilderFormValues(values); }, [values, setBuilderFormValues]); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index f46515d2835b..6f1bfc590764 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -2,11 +2,12 @@ import { useField } from "formik"; import { FormattedMessage } from "react-intl"; import { ControlLabels } from "components/LabeledControl"; +import { LabeledSwitch } from "components/LabeledSwitch"; import { DropDown } from "components/ui/DropDown"; import { Input } from "components/ui/Input"; -import { Switch } from "components/ui/Switch"; import { TagInput } from "components/ui/TagInput"; import { Text } from "components/ui/Text"; +import { InfoTooltip } from "components/ui/Tooltip/InfoTooltip"; import styles from "./BuilderField.module.scss"; @@ -63,12 +64,25 @@ export const BuilderField: React.FC = ({ path, label, tooltip const [field, meta, helpers] = useField(path); const hasError = !!meta.error && meta.touched; + if (props.type === "boolean") { + return ( + + {label} {tooltip && {tooltip}} + + } + /> + ); + } + return ( {(props.type === "text" || props.type === "number") && ( )} - {props.type === "boolean" && } {props.type === "array" && ( )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss index ee4927d9320c..1248a975e097 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -31,6 +31,7 @@ .connectorNameText { margin-left: auto; margin-right: auto; + margin-bottom: variables.$spacing-lg; } .streamsHeader { @@ -83,7 +84,7 @@ } .globalConfigButton { - margin-top: variables.$spacing-xl; + margin-top: variables.$spacing-sm; } .streamList { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss new file mode 100644 index 000000000000..2623ab06af46 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss @@ -0,0 +1,45 @@ +@use "scss/variables"; +@use "scss/colors"; + +.list { + display: flex; + flex-direction: column; + gap: variables.$spacing-md; + list-style-type: none; + padding: 0; + align-items: stretch; + margin: 0; +} + +.listItem { + display: flex; + align-items: center; +} + +.itemLabel { + flex-grow: 1; + font-size: 16px; +} + +.itemButton { + background: none !important; + border: none !important; + padding: variables.$spacing-xs; +} + +.inputForm { + gap: variables.$spacing-lg; + display: flex; + flex-direction: column; +} + +.viewContainer { + display: flex; + flex-direction: column; + align-items: center; + gap: variables.$spacing-xl; +} + +.inputsCard { + align-self: stretch; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index b532edab4a66..117d01f3ae36 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -1,17 +1,21 @@ -import { Form, Formik, useField } from "formik"; +import { faGear, faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Form, Formik, useField, useFormikContext } from "formik"; import { JSONSchema7 } from "json-schema"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { Text } from "components/ui/Text"; import { FormikPatch } from "core/form/FormikPatch"; import { BuilderFormValues } from "../types"; import { BuilderField } from "./BuilderField"; +import styles from "./InputsView.module.scss"; interface InputInEditing { key: string; @@ -21,8 +25,11 @@ interface InputInEditing { showDefaultValueField?: boolean; } +function sluggify(str: string) { + return str.toLowerCase().replaceAll(/[^a-zA-Z\d]/g, "_"); +} + export const InputsView: React.FC = () => { - const { formatMessage } = useIntl(); const [inputs, , helpers] = useField("inputs"); const [inputInEditing, setInputInEditing] = useState(undefined); const usedKeys = useMemo(() => inputs.value.map((input) => input.key), [inputs.value]); @@ -42,24 +49,35 @@ export const InputsView: React.FC = () => { [inputInEditing?.isNew, inputInEditing?.key, usedKeys] ); return ( - <> - -
    - {inputs.value.map(({ key, definition }) => ( -
  1. - + +
    2. ))}
@@ -76,6 +94,9 @@ export const InputsView: React.FC = () => { showDefaultValueField: false, }); }} + icon={} + iconPosition="left" + variant="secondary" > @@ -92,113 +113,136 @@ export const InputsView: React.FC = () => { setInputInEditing(undefined); }} > - {({ values }) => ( - <> - - } - onClose={() => { - setInputInEditing(undefined); - }} - > -
- - - - - - - - - {values.showDefaultValueField && ( - - )} - - - - - - - -
- - - )} + <> + + { + helpers.setValue(inputs.value.filter((input) => input.key !== inputInEditing.key)); + setInputInEditing(undefined); + }} + onClose={() => { + setInputInEditing(undefined); + }} + /> + )} - +
+ ); +}; +const InputModal = ({ + inputInEditing, + onClose, + onDelete, +}: { + inputInEditing: InputInEditing; + onDelete: () => void; + onClose: () => void; +}) => { + const { isValid, values, setFieldValue } = useFormikContext(); + const { formatMessage } = useIntl(); + const [title, titleMeta] = useField("definition.title"); + useEffect(() => { + if (titleMeta.touched) { + setFieldValue("key", sluggify(title.value || "")); + } + }, [setFieldValue, title.value, titleMeta.touched]); + + return ( + + } + onClose={onClose} + > +
+ + + + + + + + + {values.showDefaultValueField && ( + + )} + + + + {!inputInEditing.isNew && ( + + )} + + + +
+ ); - return <>TODO: List all inputs; }; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss index f4ddf005c44c..306a50984f99 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss @@ -1,6 +1,10 @@ @use "scss/colors"; @use "scss/variables"; +.icon { + color: colors.$grey-500; +} + .modalContent { height: 60vh; overflow: visible; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index ab47976bb996..32d3d2f33bf7 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -663,6 +663,28 @@ "connectorBuilder.uiYamlToggle.yaml": "YAML", "connectorBuilder.resetAll": "Reset all", "connectorBuilder.emptyName": "(empty)", + "connectorBuilder.inputsTitle": "User inputs", + "connectorBuilder.inputsDescription": "User inputs will be asked to the end-user in order to set up the connector.", + "connectorBuilder.addInputButton": "Add new user input", + "connectorBuilder.inputModal.newTitle": "New user input", + "connectorBuilder.inputModal.editTitle": "Edit user input", + "connectorBuilder.inputModal.inputName": "Input name", + "connectorBuilder.inputModal.inputNameTooltip": "The name of the input as it will show up in the form when configuring a connection", + "connectorBuilder.inputModal.fieldId": "Field ID", + "connectorBuilder.inputModal.fieldIdTooltip": "The ID of the field - reference it in double curly braces when referencing the value of the input somewhere in the definition of a stream or global configuration: {syntaxExample}. This value is derived automatically from the input name", + "connectorBuilder.inputModal.description": "Hint", + "connectorBuilder.inputModal.descriptionTooltip": "The hint shown in the tooltip next to the input in the form when confguring a connection", + "connectorBuilder.inputModal.type": "Type", + "connectorBuilder.inputModal.typeTooltip": "The type of the input", + "connectorBuilder.inputModal.secret": "Secret field", + "connectorBuilder.inputModal.secretTooltip": "If this option is enabled, the form input will be masked and can't be looked at again", + "connectorBuilder.inputModal.required": "Required field", + "connectorBuilder.inputModal.requiredTooltip": "If this option is enabled, the user will need to provide a value when confguring a connection", + "connectorBuilder.inputModal.showDefaultValueField": "Enable default value", + "connectorBuilder.inputModal.showDefaultValueFieldTooltip": "If a default value is provided, the input is prefilled in the configuration form", + "connectorBuilder.inputModal.default": "Default value", + "connectorBuilder.inputModal.placeholder": "Placeholder", + "connectorBuilder.inputModal.placeholderTooltip": "Placeholder shown in the form in case the user did not pick a value yet", "cloudApi.loginCallbackUrlError": "There was an error connecting to the developer portal. Please try again." } From e7d81ea1750ec05c6b94a3c1f36669447401c6f5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 14 Dec 2022 12:20:35 +0100 Subject: [PATCH 057/100] make most of inputs configuration work --- .../connectorBuilder/Builder/BuilderField.tsx | 12 ++++++++++-- .../connectorBuilder/Builder/InputsView.tsx | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 6f1bfc590764..c4ee1daa9c93 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -30,6 +30,7 @@ interface BaseFieldProps { path: string; label: string; tooltip?: string; + readOnly?: boolean; optional?: boolean; } @@ -60,7 +61,14 @@ const ArrayField: React.FC = ({ name, value, setValue, error }) return setValue(value)} error={error} />; }; -export const BuilderField: React.FC = ({ path, label, tooltip, optional = false, ...props }) => { +export const BuilderField: React.FC = ({ + path, + label, + tooltip, + optional = false, + readOnly, + ...props +}) => { const [field, meta, helpers] = useField(path); const hasError = !!meta.error && meta.touched; @@ -81,7 +89,7 @@ export const BuilderField: React.FC = ({ path, label, tooltip return ( {(props.type === "text" || props.type === "number") && ( - + )} {props.type === "array" && ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 117d01f3ae36..725cdba99a3f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -170,6 +170,7 @@ const InputModal = ({ Date: Thu, 15 Dec 2022 20:54:05 +0100 Subject: [PATCH 058/100] fix a bunch of stuff --- .../Builder/AddStreamButton.tsx | 4 +- .../connectorBuilder/Builder/BuilderField.tsx | 5 +- .../Builder/GlobalConfigView.tsx | 4 +- .../Builder/InputsView.module.scss | 4 + .../connectorBuilder/Builder/InputsView.tsx | 110 ++++++++++++------ .../Builder/StreamConfigView.tsx | 4 +- .../StreamTestingPanel/ConfigMenu.tsx | 3 + .../src/components/connectorBuilder/types.ts | 12 +- .../src/components/ui/Input/Input.module.scss | 3 +- airbyte-webapp/src/locales/en.json | 2 + .../Connector/ConnectorForm/ConnectorForm.tsx | 4 + .../Connector/ConnectorForm/FormRoot.tsx | 6 + .../components/CreateControls.module.scss | 4 + .../components/CreateControls.tsx | 12 ++ .../Connector/ConnectorForm/useBuildForm.tsx | 24 ++-- 15 files changed, 143 insertions(+), 58 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index 2177db82409c..19c6237b9747 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -66,13 +66,13 @@ export const AddStreamButton: React.FC = ({ onAddStream }) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index c4ee1daa9c93..3582fd667d18 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -36,7 +36,8 @@ interface BaseFieldProps { type BuilderFieldProps = BaseFieldProps & ( - | { type: "text" } + | { type: "string" } + | { type: "integer" } | { type: "number" } | { type: "boolean" } | { type: "array" } @@ -88,7 +89,7 @@ export const BuilderField: React.FC = ({ return ( - {(props.type === "text" || props.type === "number") && ( + {(props.type === "number" || props.type === "string" || props.type === "integer") && ( )} {props.type === "array" && ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 1d1a40a94518..62d706fd87b5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -7,14 +7,14 @@ export const GlobalConfigView: React.FC = () => { {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */} - + ); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss index 2623ab06af46..2e8f588ac3f0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss @@ -43,3 +43,7 @@ .inputsCard { align-self: stretch; } + +.deleteButtonContainer { + flex-grow: 1; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 725cdba99a3f..0505ad94981b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -13,7 +13,7 @@ import { Text } from "components/ui/Text"; import { FormikPatch } from "core/form/FormikPatch"; -import { BuilderFormValues } from "../types"; +import { BuilderFormInput } from "../types"; import { BuilderField } from "./BuilderField"; import styles from "./InputsView.module.scss"; @@ -22,15 +22,56 @@ interface InputInEditing { definition: JSONSchema7; required: boolean; isNew?: boolean; - showDefaultValueField?: boolean; + showDefaultValueField: boolean; + type: "string" | "integer" | "number" | "array" | "boolean" | "enum"; } function sluggify(str: string) { return str.toLowerCase().replaceAll(/[^a-zA-Z\d]/g, "_"); } +function newInputInEditing(): InputInEditing { + return { + key: "", + definition: {}, + required: false, + isNew: true, + showDefaultValueField: false, + type: "string", + }; +} + +function formInputToInputInEditing({ key, definition, required }: BuilderFormInput): InputInEditing { + return { + key, + definition, + required, + isNew: false, + showDefaultValueField: Boolean(definition.default), + type: definition.enum ? "enum" : (definition.type as InputInEditing["type"]), + }; +} + +function inputInEditingToFormInput({ + type, + showDefaultValueField, + isNew, + ...values +}: InputInEditing): BuilderFormInput { + return { + ...values, + definition: { + ...values.definition, + type: type === "enum" ? "string" : type, + // only respect the enum values if the user explicitly selected enum as type + enum: type === "enum" && values.definition.enum?.length ? values.definition.enum : undefined, + default: showDefaultValueField ? values.definition.default : undefined, + }, + }; +} + export const InputsView: React.FC = () => { - const [inputs, , helpers] = useField("inputs"); + const [inputs, , helpers] = useField("inputs"); const [inputInEditing, setInputInEditing] = useState(undefined); const usedKeys = useMemo(() => inputs.value.map((input) => input.key), [inputs.value]); const inputInEditValidation = useMemo( @@ -58,22 +99,16 @@ export const InputsView: React.FC = () => {
    - {inputs.value.map(({ key, definition, required }) => ( -
  1. -
    {definition.title || key}
    + {inputs.value.map((input) => ( +
  2. +
    {input.definition.title || input.key}
    +
    + +
    )} +
+ )}
{onCancel && (
+ + )} + { + setConfigString(JSON.stringify(values.connectionConfiguration, null, 2) ?? ""); + setIsOpen(false); + }} + onCancel={() => { + setIsOpen(false); + }} + submitLabel={formatMessage({ id: "connectorBuilder.saveInputsForm" })} + /> + ) : ( Date: Fri, 16 Dec 2022 14:39:17 +0100 Subject: [PATCH 061/100] fix label --- .../connectorBuilder/StreamTestingPanel/ConfigMenu.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index 36ac7b5d960e..729e325ac3eb 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -85,7 +85,7 @@ export const ConfigMenu: React.FC = ({ className }) => { onReset={() => { setConfigString("{}"); }} - submitLabel={formatMessage({ id: "connectorForm.saveInputsForm" })} + submitLabel={formatMessage({ id: "connectorBuilder.saveInputsForm" })} /> ) : ( From 75db697ebe0d9d1e1388d054ed06e63cf3ada855 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 16 Dec 2022 15:16:13 +0100 Subject: [PATCH 062/100] fix some styling --- .../Builder/InputsView.module.scss | 12 +++-- .../connectorBuilder/Builder/InputsView.tsx | 54 ++++++++++--------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss index 2e8f588ac3f0..b562a8e03b96 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss @@ -33,11 +33,13 @@ flex-direction: column; } -.viewContainer { - display: flex; - flex-direction: column; - align-items: center; - gap: variables.$spacing-xl; +.inputsDescription { + margin-top: variables.$spacing-xl; + margin-bottom: variables.$spacing-lg; +} + +.addInputButton { + align-self: center; } .inputsCard { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 0a10173127eb..0909e5774cfa 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -15,6 +15,7 @@ import { Text } from "components/ui/Text"; import { FormikPatch } from "core/form/FormikPatch"; import { BuilderFormInput } from "../types"; +import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; import styles from "./InputsView.module.scss"; @@ -75,6 +76,7 @@ function inputInEditingToFormInput({ } export const InputsView: React.FC = () => { + const { formatMessage } = useIntl(); const [inputs, , helpers] = useField("inputs"); const [inputInEditing, setInputInEditing] = useState(undefined); const usedKeys = useMemo(() => inputs.value.map((input) => input.key), [inputs.value]); @@ -94,34 +96,34 @@ export const InputsView: React.FC = () => { [inputInEditing?.isNew, inputInEditing?.key, usedKeys] ); return ( -
-

- -

- + + - -
    - {inputs.value.map((input) => ( -
  1. -
    {input.definition.title || input.key}
    - -
    2. - ))} -
- + {inputs.value.length > 0 && ( + +
    + {inputs.value.map((input) => ( +
  1. +
    {input.definition.title || input.key}
    + +
    2. + ))} +
+ + )}
+ ); }; const InputModal = ({ From 9835a5cf5affea8711c3be4c71e10cf91b922066 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 18 Dec 2022 17:38:05 +0100 Subject: [PATCH 063/100] review comments --- .../StreamTestingPanel/ConfigMenu.module.scss | 7 +- .../StreamTestingPanel/ConfigMenu.tsx | 70 +++++++++---------- .../ConfigMenuErrorBoundary.module.scss | 9 +++ .../ConfigMenuErrorBoundary.tsx | 66 +++++++++++++++++ airbyte-webapp/src/locales/en.json | 7 ++ .../ConnectorBuilderStateService.tsx | 2 +- .../Connector/ConnectorForm/ConnectorForm.tsx | 1 + .../Connector/ConnectorForm/FormRoot.tsx | 6 +- 8 files changed, 126 insertions(+), 42 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss index 179059e3d26e..d55c44c2758c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss @@ -1,10 +1,9 @@ @use "scss/colors"; @use "scss/variables"; -.modalContent { - height: 60vh; - overflow: visible; - background-color: colors.$grey-100; +.formContent { + max-height: 60vh; + overflow: auto; } .inputFormModalFooter { diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index 957a05d210b2..21f45bb9e26a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -1,19 +1,19 @@ import { faClose, faGear } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import classNames from "classnames"; import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; import { Button } from "components/ui/Button"; -import { CodeEditor } from "components/ui/CodeEditor"; import { InfoBox } from "components/ui/InfoBox"; -import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; +import { Modal, ModalBody } from "components/ui/Modal"; +import { Tooltip } from "components/ui/Tooltip"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { ConnectorForm } from "views/Connector/ConnectorForm"; import styles from "./ConfigMenu.module.scss"; +import { ConfigMenuErrorBoundaryComponent } from "./ConfigMenuErrorBoundary"; interface ConfigMenuProps { className?: string; @@ -22,7 +22,7 @@ interface ConfigMenuProps { export const ConfigMenu: React.FC = ({ className }) => { const [isOpen, setIsOpen] = useState(false); const { formatMessage } = useIntl(); - const { configString, setConfigString, jsonManifest } = useConnectorBuilderState(); + const { configString, setConfigString, jsonManifest, editorView, setEditorView } = useConnectorBuilderState(); const [showInputsWarning, setShowInputsWarning] = useLocalStorage("connectorBuilderInputsWarning", true); @@ -30,29 +30,42 @@ export const ConfigMenu: React.FC = ({ className }) => { return { connectionConfiguration: JSON.parse(configString) }; }, [configString]); - const renderInputForm = Boolean(jsonManifest.spec); + const switchToYaml = () => { + setEditorView("yaml"); + setIsOpen(false); + }; return ( <> - - - )} )} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss new file mode 100644 index 000000000000..0cc75c2324e9 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.module.scss @@ -0,0 +1,9 @@ +@use "scss/colors"; +@use "scss/variables"; + +.errorContent { + display: flex; + flex-direction: column; + gap: variables.$spacing-lg; + align-items: flex-end; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx new file mode 100644 index 000000000000..972c92c90a2e --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenuErrorBoundary.tsx @@ -0,0 +1,66 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { InfoBox } from "components/ui/InfoBox"; + +import { FormBuildError, isFormBuildError } from "core/form/FormBuildError"; +import { EditorView } from "services/connectorBuilder/ConnectorBuilderStateService"; + +import styles from "./ConfigMenuErrorBoundary.module.scss"; + +interface ApiErrorBoundaryState { + error?: string | FormBuildError; +} + +interface ApiErrorBoundaryProps { + closeAndSwitchToYaml: () => void; + currentView: EditorView; +} + +export class ConfigMenuErrorBoundaryComponent extends React.Component< + React.PropsWithChildren, + ApiErrorBoundaryState +> { + state: ApiErrorBoundaryState = {}; + + static getDerivedStateFromError(error: { message: string; __type?: string }): ApiErrorBoundaryState { + if (isFormBuildError(error)) { + return { error }; + } + + return { error: error.message }; + } + render(): React.ReactNode { + const { children, currentView, closeAndSwitchToYaml } = this.props; + const { error } = this.state; + + if (!error) { + return children; + } + return ( +
+ + }} + />{" "} + + + + + +
+ ); + } +} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index e22f1639c6af..f7f8b1917554 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -675,6 +675,13 @@ "connectorBuilder.addKeyValue": "Add", "connectorBuilder.saveInputsForm": "Save", "connectorBuilder.inputsFormWarning": "User inputs are not saved with the connector, they are required in order to test your streams, and will be asked to the end user in order to set this connector", + "connectorBuilder.inputsError": "User inputs form could not be rendered: {error}. Make sure the spec in the YAML conforms to the specified standard.", + "connectorBuilder.inputsErrorDocumentation": "Check out the documentation", + "connectorBuilder.goToYaml": "Switch to YAML view", + "connectorBuilder.close": "Close", + "connectorBuilder.inputsTooltip": "Define test inputs to check whether the connector configuration works", + "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", + "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", "jobs.noAttemptsFailure": "Failed to start job.", diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 337999cc81f7..0f4ec2c67548 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -27,7 +27,7 @@ const DEFAULT_JSON_MANIFEST_VALUES: ConnectorManifest = { streams: [], }; -type EditorView = "ui" | "yaml"; +export type EditorView = "ui" | "yaml"; export type BuilderView = "global" | number; interface Context { diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index 1bd247b911dc..90668d4c19b9 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -33,6 +33,7 @@ export interface ConnectorFormProps { successMessage?: React.ReactNode; connectorId?: string; footerClassName?: string; + bodyClassName?: string; submitLabel?: string; /** * Called in case the user cancels the form - if not provided, no cancel button is rendered diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx index d8898e6d612f..8c471a0bb268 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/FormRoot.tsx @@ -19,6 +19,7 @@ interface FormRootProps { onStopTestingConnector?: () => void; submitLabel?: string; footerClassName?: string; + bodyClassName?: string; /** * Called in case the user cancels the form - if not provided, no cancel button is rendered */ @@ -35,6 +36,7 @@ export const FormRoot: React.FC = ({ onStopTestingConnector, submitLabel, footerClassName, + bodyClassName, onCancel, }) => { const { dirty, isSubmitting, isValid } = useFormikContext(); @@ -42,7 +44,9 @@ export const FormRoot: React.FC = ({ return (
- +
+ +
{isEditMode ? ( Date: Mon, 19 Dec 2022 11:52:03 +0100 Subject: [PATCH 064/100] improve state management and error handling --- .../StreamTestingPanel/ConfigMenu.tsx | 14 ++++------ .../ConnectorBuilderStateService.tsx | 16 ++--------- .../Connector/ConnectorForm/ConnectorForm.tsx | 1 - .../Connector/ConnectorForm/useBuildForm.tsx | 27 ++++++++++++------- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index 90d83ea4413a..370505c8db5e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -1,6 +1,6 @@ import { faClose, faGear } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { useMemo, useState } from "react"; +import { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; @@ -22,14 +22,10 @@ interface ConfigMenuProps { export const ConfigMenu: React.FC = ({ className }) => { const [isOpen, setIsOpen] = useState(false); const { formatMessage } = useIntl(); - const { configString, setConfigString, jsonManifest, editorView, setEditorView } = useConnectorBuilderState(); + const { configJson, setConfigJson, jsonManifest, editorView, setEditorView } = useConnectorBuilderState(); const [showInputsWarning, setShowInputsWarning] = useLocalStorage("connectorBuilderInputsWarning", true); - const formValues = useMemo(() => { - return { connectionConfiguration: JSON.parse(configString) }; - }, [configString]); - const switchToYaml = () => { setEditorView("yaml"); setIsOpen(false); @@ -88,16 +84,16 @@ export const ConfigMenu: React.FC = ({ className }) => { bodyClassName={styles.formContent} footerClassName={styles.inputFormModalFooter} selectedConnectorDefinitionSpecification={jsonManifest.spec} - formValues={formValues} + formValues={configJson} onSubmit={async (values) => { - setConfigString(JSON.stringify(values.connectionConfiguration, null, 2) ?? ""); + setConfigJson(values); setIsOpen(false); }} onCancel={() => { setIsOpen(false); }} onReset={() => { - setConfigString("{}"); + setConfigJson({}); }} submitLabel={formatMessage({ id: "connectorBuilder.saveInputsForm" })} /> diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b3e45b699991..e143fae5e921 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -40,7 +40,6 @@ interface Context { streamListErrorMessage: string | undefined; testStreamIndex: number; selectedView: BuilderView; - configString: string; configJson: StreamReadRequestBodyConfig; editorView: EditorView; setBuilderFormValues: (values: BuilderFormValues) => void; @@ -49,7 +48,7 @@ interface Context { setYamlIsValid: (value: boolean) => void; setTestStreamIndex: (streamIndex: number) => void; setSelectedView: (view: BuilderView) => void; - setConfigString: (configString: string) => void; + setConfigJson: (value: StreamReadRequestBodyConfig) => void; setEditorView: (editorView: EditorView) => void; } @@ -86,18 +85,8 @@ export const ConnectorBuilderStateProvider: React.FC("ui"); // config - const [configString, setConfigString] = useState("{\n \n}"); const [configJson, setConfigJson] = useState({}); - useEffect(() => { - try { - const json = JSON.parse(configString) as StreamReadRequestBodyConfig; - setConfigJson(json); - } catch (err) { - console.error(`Config value is not valid JSON! Error: ${err}`); - } - }, [configString]); - // streams const { data: streamListRead, @@ -133,7 +122,6 @@ export const ConnectorBuilderStateProvider: React.FC = (props) => { async (values: ConnectorFormValues) => { const valuesToSend = getValues(values); await onSubmit(valuesToSend); - clearFormChange(formId); }, [clearFormChange, formId, getValues, onSubmit] diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index 772c086242d4..041ec792d02b 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -87,26 +87,33 @@ export function useBuildForm( throw new FormBuildError("connectorForm.error.topLevelNonObject"); } + const validationSchema = useMemo(() => buildYupFormForJsonSchema(jsonSchema, formFields), [formFields, jsonSchema]); + const startValues = useMemo(() => { - if (isEditMode) { - return { - name: "", - connectionConfiguration: {}, - ...initialValues, - }; - } - const baseValues = { + let baseValues = { name: "", connectionConfiguration: {}, ...initialValues, }; + if (isDraft) { + try { + baseValues = validationSchema.cast(baseValues, { stripUnknown: true }); + } catch { + // cast did not work which can happen if there are unexpected values in the form. Reset form in this case + baseValues.connectionConfiguration = {}; + } + } + + if (isEditMode) { + return baseValues; + } + setDefaultValues(formFields, baseValues as Record, { respectExistingValues: isDraft }); return baseValues; - }, [formFields, initialValues, isDraft, isEditMode]); + }, [formFields, initialValues, isDraft, isEditMode, validationSchema]); - const validationSchema = useMemo(() => buildYupFormForJsonSchema(jsonSchema, formFields), [formFields, jsonSchema]); return { initialValues: startValues, formFields, From 04d87cfe1a744152ba814d79665335961d5ac905 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 19 Dec 2022 17:14:43 +0100 Subject: [PATCH 065/100] allow auth configuration --- .../connectorBuilder/Builder/Builder.tsx | 6 +- .../connectorBuilder/Builder/BuilderOneOf.tsx | 66 ++++++++ .../Builder/BuilderSidebar.tsx | 7 +- .../Builder/GlobalConfigView.tsx | 66 ++++++++ .../connectorBuilder/Builder/InputsView.tsx | 147 +++++++++++------- .../Builder/UserInputField.module.scss | 12 ++ .../Builder/UserInputField.tsx | 18 +++ .../StreamTestingPanel/ConfigMenu.tsx | 5 +- .../src/components/connectorBuilder/types.ts | 98 +++++++++++- airbyte-webapp/src/locales/en.json | 1 + .../ConnectorBuilderPage.tsx | 63 ++++---- .../ConnectorBuilderStateService.tsx | 2 + 12 files changed, 397 insertions(+), 94 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index 4107fe733158..a9dbda62e6b4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -3,7 +3,7 @@ import { useEffect } from "react"; import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; -import { BuilderFormValues } from "../types"; +import { builderFormValidationSchema, BuilderFormValues } from "../types"; import styles from "./Builder.module.scss"; import { BuilderSidebar } from "./BuilderSidebar"; import { GlobalConfigView } from "./GlobalConfigView"; @@ -29,7 +29,9 @@ function getView(selectedView: BuilderView) { export const Builder: React.FC = ({ values, toggleYamlEditor }) => { const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); useEffect(() => { - setBuilderFormValues(values); + if (builderFormValidationSchema.isValidSync(values)) { + setBuilderFormValues(values); + } }, [values, setBuilderFormValues]); return ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx new file mode 100644 index 000000000000..37f339071871 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx @@ -0,0 +1,66 @@ +import { useField } from "formik"; +import React, { useEffect } from "react"; + +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; + +interface Option { + label: string; + value: string; + default?: object; +} + +interface OneOfOption { + label: string; // label shown in the dropdown menu + typeValue: string; // value to set on the `type` field for this component - should match the oneOf type definition + default?: object; // default values for the path + children?: React.ReactNode; +} + +interface BuilderOneOfProps { + options: OneOfOption[]; + path: string; // path to the oneOf component in the json schema + label: string; + tooltip: string; +} + +export const BuilderOneOf: React.FC = ({ options, path, label, tooltip }) => { + const [, , oneOfPathHelpers] = useField(path); + const typePath = `${path}.type`; + const [typePathField, , typePathHelpers] = useField(typePath); + const value = typePathField.value ?? options[0].typeValue; + + useEffect(() => { + typePathHelpers.setValue(value); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + const selectedOption = options.find((option) => option.typeValue === value); + + return ( + } + dropdown={ + { + return { label: option.label, value: option.typeValue, default: option.default }; + })} + value={value ?? options[0].typeValue} + onChange={(selectedOption: Option) => { + if (selectedOption.value === value) { + return; + } + // clear all values for this oneOf and set selected option and default values + oneOfPathHelpers.setValue({ + type: selectedOption.value, + ...selectedOption.default, + }); + }} + /> + } + > + {selectedOption?.children} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 092b84374326..3894595d67bd 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -17,7 +17,7 @@ import { } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../DownloadYamlButton"; -import { BuilderFormValues } from "../types"; +import { BuilderFormValues, getInferredInputs } from "../types"; import { useBuilderErrors } from "../useBuilderErrors"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; @@ -115,7 +115,10 @@ export const BuilderSidebar: React.FC = ({ className, toggl onClick={() => handleViewSelect("inputs")} > - +
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 433d2d2e5296..43c659f06797 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -3,8 +3,10 @@ import { useIntl } from "react-intl"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; +import { BuilderOneOf } from "./BuilderOneOf"; import { BuilderTitle } from "./BuilderTitle"; import styles from "./GlobalConfigView.module.scss"; +import { UserInputField } from "./UserInputField"; export const GlobalConfigView: React.FC = () => { const { formatMessage } = useIntl(); @@ -16,6 +18,70 @@ export const GlobalConfigView: React.FC = () => { + + + + + + ), + }, + { + label: "Bearer", + typeValue: "BearerAuthenticator", + default: { + api_token: "{{ config['api_key'] }}", + }, + children: ( + + ), + }, + { + label: "Basic HTTP", + typeValue: "BasicHttpAuthenticator", + default: { + username: "{{ config['username'] }}", + password: "{{ config['password'] }}", + }, + children: ( + <> + + + + ), + }, + ]} + /> + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 0909e5774cfa..755e7f0d7637 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -14,7 +14,7 @@ import { Text } from "components/ui/Text"; import { FormikPatch } from "core/form/FormikPatch"; -import { BuilderFormInput } from "../types"; +import { BuilderFormInput, BuilderFormValues, getInferredInputs } from "../types"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; import styles from "./InputsView.module.scss"; @@ -28,6 +28,7 @@ interface InputInEditing { isNew?: boolean; showDefaultValueField: boolean; type: typeof supportedTypes[number]; + isInferredInputOverride: boolean; } function sluggify(str: string) { @@ -42,10 +43,14 @@ function newInputInEditing(): InputInEditing { isNew: true, showDefaultValueField: false, type: "string", + isInferredInputOverride: false, }; } -function formInputToInputInEditing({ key, definition, required }: BuilderFormInput): InputInEditing { +function formInputToInputInEditing( + { key, definition, required }: BuilderFormInput, + isInferredInputOverride: boolean +): InputInEditing { const supportedType = supportedTypes.find((type) => type === definition.type) || "unknown"; return { key, @@ -54,6 +59,7 @@ function formInputToInputInEditing({ key, definition, required }: BuilderFormInp isNew: false, showDefaultValueField: Boolean(definition.default), type: supportedType !== "unknown" && definition.enum ? "enum" : supportedType, + isInferredInputOverride, }; } @@ -77,6 +83,7 @@ function inputInEditingToFormInput({ export const InputsView: React.FC = () => { const { formatMessage } = useIntl(); + const { values, setFieldValue } = useFormikContext(); const [inputs, , helpers] = useField("inputs"); const [inputInEditing, setInputInEditing] = useState(undefined); const usedKeys = useMemo(() => inputs.value.map((input) => input.key), [inputs.value]); @@ -95,33 +102,52 @@ export const InputsView: React.FC = () => { }), [inputInEditing?.isNew, inputInEditing?.key, usedKeys] ); + const inferredInputs = getInferredInputs(values); + return ( - {inputs.value.length > 0 && ( - -
    - {inputs.value.map((input) => ( -
  1. -
    {input.definition.title || input.key}
    - -
    2. - ))} -
- - )} + {inputs.value.length > 0 || + (inferredInputs.length > 0 && ( + +
    + {inferredInputs.map((input) => ( +
  1. +
    {input.definition.title || input.key}
    + +
    2. + ))} + {inputs.value.map((input) => ( +
  2. +
    {input.definition.title || input.key}
    + +
    3. + ))} +
+ + ))} - - ))} - {inputs.value.map((input) => ( -
  • -
    {input.definition.title || input.key}
    - -
    • - ))} - - - ))} + {(inputs.value.length > 0 || inferredInputs.length > 0) && ( + +
      + {inferredInputs.map((input) => ( +
    1. +
      {input.definition.title || input.key}
      + +
      2. + ))} + {inputs.value.map((input) => ( +
    2. +
      {input.definition.title || input.key}
      + +
      3. + ))} +
    +     + )} + {isOpen &&
    {children}
    } +
    + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 43c659f06797..bc8e29a532a5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -1,12 +1,11 @@ import { useIntl } from "react-intl"; +import { AuthenticationSection } from "./AuthenticationSection"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; -import { BuilderOneOf } from "./BuilderOneOf"; import { BuilderTitle } from "./BuilderTitle"; import styles from "./GlobalConfigView.module.scss"; -import { UserInputField } from "./UserInputField"; export const GlobalConfigView: React.FC = () => { const { formatMessage } = useIntl(); @@ -18,70 +17,7 @@ export const GlobalConfigView: React.FC = () => { - - - - - - ), - }, - { - label: "Bearer", - typeValue: "BearerAuthenticator", - default: { - api_token: "{{ config['api_key'] }}", - }, - children: ( - - ), - }, - { - label: "Basic HTTP", - typeValue: "BasicHttpAuthenticator", - default: { - username: "{{ config['username'] }}", - password: "{{ config['password'] }}", - }, - children: ( - <> - - - - ), - }, - ]} - /> - + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index ea1af0428cff..723f9853cf0e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -4,7 +4,16 @@ import * as yup from "yup"; import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { AirbyteJSONSchema } from "core/jsonSchema/types"; -import { DeclarativeStream, HttpRequesterAllOfAuthenticator } from "core/request/ConnectorManifest"; +import { + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + DeclarativeOauth2AuthenticatorAllOf, + DeclarativeStream, + HttpRequesterAllOfAuthenticator, + NoAuth, + SessionTokenAuthenticator, +} from "core/request/ConnectorManifest"; export interface BuilderFormInput { key: string; @@ -12,11 +21,22 @@ export interface BuilderFormInput { definition: AirbyteJSONSchema; } +type BuilderFormAuthenticator = ( + | NoAuth + | (Omit & { + refresh_request_body: Array<[string, string]>; + }) + | ApiKeyAuthenticator + | BearerAuthenticator + | BasicHttpAuthenticator + | SessionTokenAuthenticator +) & { type: string }; + export interface BuilderFormValues { global: { connectorName: string; urlBase: string; - authenticator: HttpRequesterAllOfAuthenticator; + authenticator: BuilderFormAuthenticator; }; inputs: BuilderFormInput[]; inferredInputOverrides: Record>; @@ -82,6 +102,68 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] { }, ]; } + if (values.global.authenticator.type === "OAuthAuthenticator") { + return [ + { + key: "client_id", + required: true, + definition: { + type: "string", + title: "Client ID", + airbyte_secret: true, + }, + }, + { + key: "client_secret", + required: true, + definition: { + type: "string", + title: "Client secret", + airbyte_secret: true, + }, + }, + { + key: "refresh_token", + required: true, + definition: { + type: "string", + title: "Refresh token", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "SessionTokenAuthenticator") { + return [ + { + key: "username", + required: false, + definition: { + type: "string", + title: "Username", + }, + }, + { + key: "password", + required: false, + definition: { + type: "string", + title: "Password", + airbyte_secret: true, + }, + }, + { + key: "session_token", + required: false, + definition: { + type: "string", + title: "Session token", + description: "Session token generated by user (if provided username and password are not required)", + airbyte_secret: true, + }, + }, + ]; + } return []; } @@ -103,7 +185,7 @@ export const builderFormValidationSchema = yup.object().shape({ urlBase: yup.string().required("form.empty.error"), authenticator: yup.object({ header: yup.mixed().when("type", { - is: "ApiKeyAuthenticator", + is: (type: string) => type === "ApiKeyAuthenticator" || type === "SessionTokenAuthenticator", then: yup.string().required("form.empty.error"), otherwise: (schema) => schema.strip(), }), @@ -122,6 +204,26 @@ export const builderFormValidationSchema = yup.object().shape({ then: yup.string().required("form.empty.error"), otherwise: (schema) => schema.strip(), }), + token_refresh_endpoint: yup.mixed().when("type", { + is: "OAuthAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + session_token_response_key: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + login_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + validate_session_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), }), }), streams: yup.array().of( @@ -139,6 +241,24 @@ export const builderFormValidationSchema = yup.object().shape({ ), }); +function builderFormAuthenticatorToAuthenticator( + globalSettings: BuilderFormValues["global"] +): HttpRequesterAllOfAuthenticator { + if (globalSettings.authenticator.type === "OAuthAuthenticator") { + return { + ...globalSettings.authenticator, + refresh_request_body: Object.fromEntries(globalSettings.authenticator.refresh_request_body), + }; + } + if (globalSettings.authenticator.type === "SessionTokenAuthenticator") { + return { + ...globalSettings.authenticator, + api_url: globalSettings.urlBase, + }; + } + return globalSettings.authenticator as HttpRequesterAllOfAuthenticator; +} + export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { @@ -154,7 +274,7 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), request_body_data: Object.fromEntries(stream.requestOptions.requestBody), }, - authenticator: values.global.authenticator, + authenticator: builderFormAuthenticatorToAuthenticator(values.global), // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema config: {}, }, diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index df4641dbe32c..47d946bc15b3 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -716,6 +716,7 @@ "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel", + "connectorBuilder.optionalFieldsLabel": "Optional fields", "jobs.noAttemptsFailure": "Failed to start job.", From 7b6470e53b844f3242f554e2d8a061a1b15312c6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 20 Dec 2022 15:20:01 +0100 Subject: [PATCH 070/100] fill in session token variable --- .../connectorBuilder/Builder/AuthenticationSection.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx index d49037eb452b..5872449cd93c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx @@ -133,7 +133,7 @@ export const AuthenticationSection: React.FC = () => { default: { username: "{{ config['username'] }}", password: "{{ config['password'] }}", - session_token: "", + session_token: "{{ config['session_token'] }}", }, children: ( <> From 23f4fbe746b03f563f1050a36cf4904adb5befe3 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 19 Dec 2022 16:36:44 -0800 Subject: [PATCH 071/100] fix merge of default values --- .../services/connectorBuilder/ConnectorBuilderStateService.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b5b7b9818c2a..b979f12e7c52 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -1,4 +1,5 @@ import { dump } from "js-yaml"; +import merge from "lodash/merge"; import React, { useContext, useEffect, useMemo, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; @@ -65,7 +66,7 @@ export const ConnectorBuilderStateProvider: React.FC { - return { ...DEFAULT_BUILDER_FORM_VALUES, ...(storedBuilderFormValues ?? {}) }; + return merge(DEFAULT_BUILDER_FORM_VALUES, storedBuilderFormValues); }, [storedBuilderFormValues]); const [jsonManifest, setJsonManifest] = useLocalStorage( From d4975bfbe952fc0fce11832c6fe18b41eaa4693c Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 19 Dec 2022 16:53:30 -0800 Subject: [PATCH 072/100] add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts --- .../Builder/AddStreamButton.tsx | 15 +++------ .../Builder/BuilderSidebar.tsx | 8 ++--- .../src/components/connectorBuilder/types.ts | 32 +++++++++++++++++++ .../ConnectorBuilderStateService.tsx | 13 +------- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index f4a72b90aa35..be8ed5a1816d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -1,4 +1,5 @@ import { Form, Formik, useField } from "formik"; +import merge from "lodash/merge"; import { useState } from "react"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; @@ -9,7 +10,7 @@ import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; import { FormikPatch } from "core/form/FormikPatch"; import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg"; -import { BuilderStream } from "../types"; +import { BuilderStream, DEFAULT_BUILDER_STREAM_VALUES } from "../types"; import styles from "./AddStreamButton.module.scss"; import { BuilderField } from "./BuilderField"; @@ -49,18 +50,10 @@ export const AddStreamButton: React.FC = ({ onAddStream, b onSubmit={(values: AddStreamValues) => { helpers.setValue([ ...streamsField.value, - { - fieldPointer: [], - httpMethod: "GET", - requestOptions: { - requestParameters: [], - requestHeaders: [], - requestBody: [], - }, - ...initialValues, + merge({}, DEFAULT_BUILDER_STREAM_VALUES, { name: values.streamName, urlPath: values.urlPath, - }, + }), ]); setIsOpen(false); onAddStream(numStreams); diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 3894595d67bd..7bd54599ea6b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -10,14 +10,10 @@ import { Heading } from "components/ui/Heading"; import { Text } from "components/ui/Text"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { - BuilderView, - DEFAULT_BUILDER_FORM_VALUES, - useConnectorBuilderState, -} from "services/connectorBuilder/ConnectorBuilderStateService"; +import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../DownloadYamlButton"; -import { BuilderFormValues, getInferredInputs } from "../types"; +import { BuilderFormValues, DEFAULT_BUILDER_FORM_VALUES, getInferredInputs } from "../types"; import { useBuilderErrors } from "../useBuilderErrors"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 723f9853cf0e..c0f696ec1a83 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -47,6 +47,8 @@ export interface BuilderStream { name: string; urlPath: string; fieldPointer: string[]; + primaryKey: string[]; + cursorField: string[]; httpMethod: "GET" | "POST"; requestOptions: { requestParameters: Array<[string, string]>; @@ -54,6 +56,32 @@ export interface BuilderStream { requestBody: Array<[string, string]>; }; } + +export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { + global: { + connectorName: "", + urlBase: "", + authenticator: { type: "NoAuth" }, + }, + inputs: [], + inferredInputOverrides: {}, + streams: [], +}; + +export const DEFAULT_BUILDER_STREAM_VALUES: BuilderStream = { + name: "", + urlPath: "", + fieldPointer: [], + primaryKey: [], + cursorField: [], + httpMethod: "GET", + requestOptions: { + requestParameters: [], + requestHeaders: [], + requestBody: [], + }, +}; + function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] { if (values.global.authenticator.type === "ApiKeyAuthenticator") { return [ @@ -231,6 +259,8 @@ export const builderFormValidationSchema = yup.object().shape({ name: yup.string().required("form.empty.error"), urlPath: yup.string().required("form.empty.error"), fieldPointer: yup.array().of(yup.string()), + cursorField: yup.array().of(yup.string()), + primaryKey: yup.array().of(yup.string()), httpMethod: yup.mixed().oneOf(["GET", "POST"]), requestOptions: yup.object().shape({ requestParameters: yup.array().of(yup.array().of(yup.string())), @@ -263,8 +293,10 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { name: stream.name, + stream_cursor_field: stream.cursorField, retriever: { name: stream.name, + primary_key: stream.primaryKey, requester: { name: stream.name, url_base: values.global?.urlBase, diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b979f12e7c52..9f4046abbb00 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -4,24 +4,13 @@ import React, { useContext, useEffect, useMemo, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; -import { BuilderFormValues, convertToManifest } from "components/connectorBuilder/types"; +import { BuilderFormValues, convertToManifest, DEFAULT_BUILDER_FORM_VALUES } from "components/connectorBuilder/types"; import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { StreamReadRequestBodyConfig, StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { useListStreams } from "./ConnectorBuilderApiService"; -export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { - global: { - connectorName: "", - urlBase: "", - authenticator: { type: "NoAuth" }, - }, - inputs: [], - inferredInputOverrides: {}, - streams: [], -}; - const DEFAULT_JSON_MANIFEST_VALUES: PatchedConnectorManifest = { version: "0.1.0", check: { From 32213f88196d50c8068d585315b900f7129ae972 Mon Sep 17 00:00:00 2001 From: lmossman Date: Mon, 19 Dec 2022 17:57:34 -0800 Subject: [PATCH 073/100] add cursor and primary key fields to ui --- .../connectorBuilder/Builder/StreamConfigView.tsx | 14 +++++++++++++- .../src/components/connectorBuilder/types.ts | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index f346fbfd4819..05fa4b79ebdf 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -81,10 +81,22 @@ export const StreamConfigView: React.FC = ({ streamNum }) label="HTTP Method" tooltip="HTTP method to use for requests sent to the API" /> + + diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index c0f696ec1a83..9c7be635b5f2 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -294,6 +294,7 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa return { name: stream.name, stream_cursor_field: stream.cursorField, + primary_key: stream.primaryKey, retriever: { name: stream.name, primary_key: stream.primaryKey, From 3e203243f251b8c4140c3d8350733ad3d2c558dd Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 11:08:11 -0800 Subject: [PATCH 074/100] save --- .../Builder/StreamConfigView.tsx | 3 ++ .../src/components/connectorBuilder/types.ts | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 05fa4b79ebdf..e925afbd6694 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -117,6 +117,9 @@ export const StreamConfigView: React.FC = ({ streamNum }) tooltip="Body to attach to API requests as url-encoded form values" /> + {/* + + */} ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 9c7be635b5f2..a5f6b808aa25 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -13,6 +13,8 @@ import { HttpRequesterAllOfAuthenticator, NoAuth, SessionTokenAuthenticator, + DefaultPaginatorAllOfPaginationStrategy, + RequestOption, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -55,6 +57,11 @@ export interface BuilderStream { requestHeaders: Array<[string, string]>; requestBody: Array<[string, string]>; }; + paginator?: { + strategy: DefaultPaginatorAllOfPaginationStrategy; + pageSizeOption: RequestOption; + pageTokenOption: RequestOption; + }; } export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { @@ -207,6 +214,12 @@ export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] ); } +export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; +const requestOptionSchema = yup.object().shape({ + inject_into: yup.mixed().oneOf(injectIntoValues), + field_name: yup.string(), +}); + export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ connectorName: yup.string().required("form.empty.error"), @@ -267,6 +280,32 @@ export const builderFormValidationSchema = yup.object().shape({ requestHeaders: yup.array().of(yup.array().of(yup.string())), requestBody: yup.array().of(yup.array().of(yup.string())), }), + paginator: yup.object().shape({ + strategy: yup.object().shape({ + page_size: yup.mixed().when("type", { + is: (val: string) => ["CursorPagination", "OffsetIncrement", "PageIncrement"].includes(val), + then: yup.number().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + cursor_value: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + stop_condition: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + start_from_page: yup.mixed().when("type", { + is: "PageIncrement", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }), + pageSizeOption: requestOptionSchema, + pageTokenOption: requestOptionSchema, + }), }) ), }); From 3ee0ffb8b6f506e24be559795a586601a035f76a Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 17:52:11 -0800 Subject: [PATCH 075/100] add page size and token option inputs --- .../Builder/Builder.module.scss | 1 + .../Builder/PaginationSection.tsx | 75 +++++++++++++++++++ .../Builder/StreamConfigView.tsx | 11 ++- .../src/components/connectorBuilder/types.ts | 71 ++++++++++-------- 4 files changed, 122 insertions(+), 36 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss index 4c361af3aa4e..8ece94e80807 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss @@ -18,4 +18,5 @@ .form { flex: 1; padding: variables.$spacing-xl; + overflow: auto; } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx new file mode 100644 index 000000000000..635ae3cc3d7a --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -0,0 +1,75 @@ +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; + +import { injectIntoValues } from "../types"; +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; + +interface PaginationSectionProps { + streamFieldPath: (fieldPath: string) => string; +} + +export const PaginationSection: React.FC = ({ streamFieldPath }) => { + const pageTokenOption = ( + + } + > + + + + ); + + const pageSizeOption = ( + + } + > + + + + ); + + return ( + + {pageTokenOption} + {pageSizeOption} + + {/* */} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index e925afbd6694..8398d5242318 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -14,6 +14,7 @@ import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; import { BuilderTitle } from "./BuilderTitle"; import { KeyValueListField } from "./KeyValueListField"; +import { PaginationSection } from "./PaginationSection"; import styles from "./StreamConfigView.module.scss"; interface StreamConfigViewProps { @@ -84,19 +85,19 @@ export const StreamConfigView: React.FC = ({ streamNum }) @@ -117,9 +118,7 @@ export const StreamConfigView: React.FC = ({ streamNum }) tooltip="Body to attach to API requests as url-encoded form values" /> - {/* - - */} + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index a5f6b808aa25..f4752912544b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -215,10 +215,14 @@ export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] } export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; -const requestOptionSchema = yup.object().shape({ - inject_into: yup.mixed().oneOf(injectIntoValues), - field_name: yup.string(), -}); +const requestOptionSchema = yup + .object() + .shape({ + inject_into: yup.mixed().oneOf(injectIntoValues), + field_name: yup.string(), + }) + .notRequired() + .default(undefined); export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ @@ -280,32 +284,39 @@ export const builderFormValidationSchema = yup.object().shape({ requestHeaders: yup.array().of(yup.array().of(yup.string())), requestBody: yup.array().of(yup.array().of(yup.string())), }), - paginator: yup.object().shape({ - strategy: yup.object().shape({ - page_size: yup.mixed().when("type", { - is: (val: string) => ["CursorPagination", "OffsetIncrement", "PageIncrement"].includes(val), - then: yup.number().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - cursor_value: yup.mixed().when("type", { - is: "CursorPagination", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - stop_condition: yup.mixed().when("type", { - is: "CursorPagination", - then: yup.string(), - otherwise: (schema) => schema.strip(), - }), - start_from_page: yup.mixed().when("type", { - is: "PageIncrement", - then: yup.string(), - otherwise: (schema) => schema.strip(), - }), - }), - pageSizeOption: requestOptionSchema, - pageTokenOption: requestOptionSchema, - }), + paginator: yup + .object() + .shape({ + pageSizeOption: requestOptionSchema, + pageTokenOption: requestOptionSchema, + strategy: yup + .object({ + page_size: yup.mixed().when("type", { + is: (val: string) => ["CursorPagination", "OffsetIncrement", "PageIncrement"].includes(val), + then: yup.number().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + cursor_value: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + stop_condition: yup.mixed().when("type", { + is: "CursorPagination", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + start_from_page: yup.mixed().when("type", { + is: "PageIncrement", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }) + .notRequired() + .default(undefined), + }) + .notRequired() + .default(undefined), }) ), }); From 0576478c37bb81f62ed73fd2c4cf864d75462c22 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 17:57:13 -0800 Subject: [PATCH 076/100] fixes after rebase --- .../src/components/connectorBuilder/Builder/AddStreamButton.tsx | 1 + .../services/connectorBuilder/ConnectorBuilderStateService.tsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx index be8ed5a1816d..73c9b18d4d1b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -51,6 +51,7 @@ export const AddStreamButton: React.FC = ({ onAddStream, b helpers.setValue([ ...streamsField.value, merge({}, DEFAULT_BUILDER_STREAM_VALUES, { + ...initialValues, name: values.streamName, urlPath: values.urlPath, }), diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 9f4046abbb00..0b43cd29c672 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -55,7 +55,7 @@ export const ConnectorBuilderStateProvider: React.FC { - return merge(DEFAULT_BUILDER_FORM_VALUES, storedBuilderFormValues); + return merge({}, DEFAULT_BUILDER_FORM_VALUES, storedBuilderFormValues); }, [storedBuilderFormValues]); const [jsonManifest, setJsonManifest] = useLocalStorage( From 5e45f59b8e77ba18dfeae33c80da9fe3e7c3785f Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 19:02:09 -0800 Subject: [PATCH 077/100] add pagination --- .../Builder/BuilderCard.module.scss | 5 + .../connectorBuilder/Builder/BuilderCard.tsx | 29 ++++- .../Builder/PaginationSection.tsx | 119 ++++++++++++++++-- .../Builder/StreamConfigView.tsx | 2 +- .../src/components/connectorBuilder/types.ts | 14 ++- 5 files changed, 157 insertions(+), 12 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss index 48fedff9a2a0..552148a02cf6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss @@ -6,3 +6,8 @@ flex-direction: column; gap: variables.$spacing-xl; } + +.toggleContainer { + display: flex; + gap: variables.$spacing-md; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx index 312884a7a2b4..deec879b8c91 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx @@ -2,13 +2,38 @@ import classNames from "classnames"; import React from "react"; import { Card } from "components/ui/Card"; +import { CheckBox } from "components/ui/CheckBox"; import styles from "./BuilderCard.module.scss"; interface BuilderCardProps { className?: string; + toggleConfig?: { + label: React.ReactNode; + toggledOn: boolean; + onToggle: (newToggleValue: boolean) => void; + }; } -export const BuilderCard: React.FC> = ({ children, className }) => { - return {children}; +export const BuilderCard: React.FC> = ({ + children, + className, + toggleConfig, +}) => { + return ( + + {toggleConfig && ( +
    + { + toggleConfig.onToggle(event.target.checked); + }} + /> + {toggleConfig.label} +
    + )} + {(!toggleConfig || toggleConfig.toggledOn) && children} +     + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 635ae3cc3d7a..672a0f817bfa 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -1,15 +1,39 @@ +import { useField } from "formik"; + import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; import { injectIntoValues } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; +import { BuilderOneOf } from "./BuilderOneOf"; interface PaginationSectionProps { streamFieldPath: (fieldPath: string) => string; } export const PaginationSection: React.FC = ({ streamFieldPath }) => { + const [field, , helpers] = useField(streamFieldPath("paginator")); + + const handleToggle = (newToggleValue: boolean) => { + if (newToggleValue) { + helpers.setValue({ + strategy: { + type: "OffsetIncrement", + }, + pageSizeOption: { + inject_into: "request_parameter", + }, + pageTokenOption: { + inject_into: "request_parameter", + }, + }); + } else { + helpers.setValue(undefined); + } + }; + const toggledOn = field.value !== undefined; + const pageTokenOption = ( = ({ streamFiel ); return ( - - {pageTokenOption} - {pageSizeOption} - - {/* */} + + ), + toggledOn, + onToggle: handleToggle, + }} + > + + + {pageTokenOption} + {pageSizeOption} + + ), + }, + { + label: "Page Increment", + typeValue: "PageIncrement", + children: ( + <> + + + {pageTokenOption} + {pageSizeOption} + + ), + }, + { + label: "Cursor Pagination", + typeValue: "CursorPagination", + children: ( + <> + + + + {pageTokenOption} + {pageSizeOption} + + ), + }, + ]} + /> ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 8398d5242318..2282992f0577 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -101,6 +101,7 @@ export const StreamConfigView: React.FC = ({ streamNum }) tooltip="Pointer into the response that should be extracted as the final record" /> + = ({ streamNum }) tooltip="Body to attach to API requests as url-encoded form values" /> - ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index f4752912544b..eaf62d670c97 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -292,9 +292,9 @@ export const builderFormValidationSchema = yup.object().shape({ strategy: yup .object({ page_size: yup.mixed().when("type", { - is: (val: string) => ["CursorPagination", "OffsetIncrement", "PageIncrement"].includes(val), + is: (val: string) => ["OffsetIncrement", "PageIncrement"].includes(val), then: yup.number().required("form.empty.error"), - otherwise: (schema) => schema.strip(), + otherwise: yup.number(), }), cursor_value: yup.mixed().when("type", { is: "CursorPagination", @@ -367,6 +367,16 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa config: {}, }, }, + paginator: stream.paginator + ? { + page_token_option: stream.paginator.pageTokenOption, + page_size_option: stream.paginator.pageSizeOption, + pagination_strategy: { + ...stream.paginator.strategy, + url_base: values.global?.urlBase, + }, + } + : { type: "NoPagination" }, config: {}, }, config: {}, From f101602a4b8a7de33757721c5cd077e51df9178f Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 20:41:30 -0800 Subject: [PATCH 078/100] fix pagination types --- airbyte-webapp/src/components/connectorBuilder/types.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index eaf62d670c97..e7e0889df887 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -369,12 +369,11 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa }, paginator: stream.paginator ? { + type: "DefaultPaginator", page_token_option: stream.paginator.pageTokenOption, page_size_option: stream.paginator.pageSizeOption, - pagination_strategy: { - ...stream.paginator.strategy, - url_base: values.global?.urlBase, - }, + pagination_strategy: stream.paginator.strategy, + url_base: values.global?.urlBase, } : { type: "NoPagination" }, config: {}, From 5fb62e2fdb8918609489f3dc33ce67fd2fb8add2 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 20:59:36 -0800 Subject: [PATCH 079/100] handle empty field_name better --- airbyte-webapp/src/components/connectorBuilder/types.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index e7e0889df887..5406b3d7e59d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -370,7 +370,13 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa paginator: stream.paginator ? { type: "DefaultPaginator", - page_token_option: stream.paginator.pageTokenOption, + page_token_option: { + ...stream.paginator.pageTokenOption, + // ensures that empty field_name is not set, as connector builder server cannot accept a field_name if inject_into is set to 'path' + field_name: stream.paginator.pageTokenOption.field_name + ? stream.paginator.pageTokenOption.field_name + : undefined, + }, page_size_option: stream.paginator.pageSizeOption, pagination_strategy: stream.paginator.strategy, url_base: values.global?.urlBase, From 7cce3502073e1c72861ee1a793aae0d1ed5ce00d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 10:49:43 +0100 Subject: [PATCH 080/100] Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman --- airbyte-webapp/src/locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 2d5de27085f0..5086c73a6e0c 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -678,7 +678,7 @@ "connectorBuilder.uiYamlToggle.yaml": "YAML", "connectorBuilder.resetAll": "Reset all", "connectorBuilder.emptyName": "(empty)", - "connectorBuilder.inputsTitle": "User inputs", + "connectorBuilder.inputsTitle": "User Inputs", "connectorBuilder.inputsDescription": "User inputs will be asked to the end-user in order to set up the connector.", "connectorBuilder.addInputButton": "Add new user input", "connectorBuilder.inputModal.newTitle": "New user input", From 20e8044272fece908237a4b15a147ebb050dbcf6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 10:49:56 +0100 Subject: [PATCH 081/100] Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman --- .../src/components/connectorBuilder/Builder/InputsView.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 0909e5774cfa..eec0c33570f0 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -165,6 +165,7 @@ export const InputsView: React.FC = () => { ); }; + const InputModal = ({ inputInEditing, onClose, From 50bddcbb423b4966cf282c71d3c2978d3df4d6bc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 11:46:58 +0100 Subject: [PATCH 082/100] inputs editing weirdness --- .../connectorBuilder/Builder/BuilderField.tsx | 33 ++++++++++++++----- .../connectorBuilder/Builder/InputsView.tsx | 29 +++++++++------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index c3d92c71d7c8..1bffdfcc8d72 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -35,7 +35,12 @@ interface BaseFieldProps { } type BuilderFieldProps = BaseFieldProps & - ({ type: "string" | "integer" | "number" | "boolean" | "array" } | { type: "enum"; options: string[] }); + ( + | { type: "string" | "number" | "integer"; onChange?: (newValue: string) => void } + | { type: "boolean"; onChange?: (newValue: boolean) => void } + | { type: "array"; onChange?: (newValue: string[]) => void } + | { type: "enum"; onChange?: (newValue: string) => void; options: string[] } + ); const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { return ( @@ -80,21 +85,31 @@ export const BuilderField: React.FC = ({ ); } + const setValue = (newValue: unknown) => { + props.onChange?.(newValue as string & string[]); + helpers.setValue(newValue); + }; + return ( {(props.type === "number" || props.type === "string" || props.type === "integer") && ( - + { + field.onChange(e); + props.onChange?.(e.target.value); + }} + type={props.type} + value={field.value ?? ""} + error={hasError} + readOnly={readOnly} + /> )} {props.type === "array" && ( - + )} {props.type === "enum" && ( - + )} {hasError && ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index eec0c33570f0..85f15053b4eb 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -2,8 +2,9 @@ import { faGear, faPlus } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Form, Formik, useField, useFormikContext } from "formik"; import { JSONSchema7 } from "json-schema"; -import { useEffect, useMemo, useState } from "react"; +import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; +import { useEffectOnce } from "react-use"; import * as yup from "yup"; import { Button } from "components/ui/Button"; @@ -86,8 +87,10 @@ export const InputsView: React.FC = () => { // make sure key can only occur once key: yup .string() - .required("form.empty.error") - .notOneOf(inputInEditing?.isNew ? usedKeys : usedKeys.filter((key) => key !== inputInEditing?.key)), + .notOneOf( + inputInEditing?.isNew ? usedKeys : usedKeys.filter((key) => key !== inputInEditing?.key), + "connectorBuilder.duplicateFieldID" + ), required: yup.bool(), definition: yup.object().shape({ title: yup.string().required("form.empty.error"), @@ -175,14 +178,12 @@ const InputModal = ({ onDelete: () => void; onClose: () => void; }) => { - const { isValid, values, setFieldValue } = useFormikContext(); + const { isValid, values, setFieldValue, setTouched } = useFormikContext(); const { formatMessage } = useIntl(); - const [title, titleMeta] = useField("definition.title"); - useEffect(() => { - if (titleMeta.touched) { - setFieldValue("key", sluggify(title.value || "")); - } - }, [setFieldValue, title.value, titleMeta.touched]); + useEffectOnce(() => { + // key input is always touched so errors are shown right away as it will be auto-set by the user changing the title + setTouched({ key: true }); + }); return ( { + setFieldValue("key", sluggify(newValue || ""), true); + }} label={formatMessage({ id: "connectorBuilder.inputModal.inputName" })} tooltip={formatMessage({ id: "connectorBuilder.inputModal.inputNameTooltip" })} /> @@ -210,7 +214,7 @@ const InputModal = ({ tooltip={formatMessage( { id: "connectorBuilder.inputModal.fieldIdTooltip" }, { - syntaxExample: "{{my_input}}", + syntaxExample: `{{config['${values.key || "my_input"}']}}`, } )} /> @@ -227,6 +231,9 @@ const InputModal = ({ path="type" type="enum" options={["string", "number", "integer", "array", "boolean", "enum"]} + onChange={() => { + setFieldValue("definition.default", undefined); + }} label={formatMessage({ id: "connectorBuilder.inputModal.type" })} tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} /> From 0ca37ad2b9d7197f2a5c5a786e20c9e067c6d491 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 11:47:19 +0100 Subject: [PATCH 083/100] input form reset --- .../connectorBuilder/StreamTestingPanel/ConfigMenu.tsx | 5 +++-- airbyte-webapp/src/locales/en.json | 1 + .../views/Connector/ConnectorForm/ConnectorForm.tsx | 10 +++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index 370505c8db5e..d2560c251ec7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -9,6 +9,7 @@ import { InfoBox } from "components/ui/InfoBox"; import { Modal, ModalBody } from "components/ui/Modal"; import { Tooltip } from "components/ui/Tooltip"; +import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { ConnectorForm } from "views/Connector/ConnectorForm"; @@ -84,9 +85,9 @@ export const ConfigMenu: React.FC = ({ className }) => { bodyClassName={styles.formContent} footerClassName={styles.inputFormModalFooter} selectedConnectorDefinitionSpecification={jsonManifest.spec} - formValues={configJson} + formValues={{ connectionConfiguration: configJson }} onSubmit={async (values) => { - setConfigJson(values); + setConfigJson(values.connectionConfiguration as StreamReadRequestBodyConfig); setIsOpen(false); }} onCancel={() => { diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 5086c73a6e0c..c55bc6a2d1c2 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -715,6 +715,7 @@ "connectorBuilder.inputsTooltip": "Define test inputs to check whether the connector configuration works", "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", + "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times", "jobs.noAttemptsFailure": "Failed to start job.", diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index 758568d284c1..f193a3f41e8c 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -64,6 +64,7 @@ export const ConnectorForm: React.FC = (props) => { selectedConnectorDefinitionSpecification, errorMessage, connectorId, + onReset, } = props; const { formFields, initialValues, validationSchema } = useBuildForm( @@ -99,7 +100,7 @@ export const ConnectorForm: React.FC = (props) => { onSubmit={onFormSubmit} enableReinitialize > - {({ dirty }) => ( + {({ dirty, resetForm }) => ( = (props) => { {...props} formFields={formFields} errorMessage={errorMessage} + onReset={ + onReset && + (() => { + onReset?.(); + resetForm(); + }) + } onStopTestingConnector={onStopTesting ? () => onStopTesting() : undefined} onRetest={testConnector ? async () => await testConnector() : undefined} /> From 8095fda2e0bc6957a489762e2c7ec01892e3546f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 11:49:04 +0100 Subject: [PATCH 084/100] using the Label component --- .../components/connectorBuilder/Builder/InputsView.module.scss | 1 - .../src/components/connectorBuilder/Builder/InputsView.tsx | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss index b562a8e03b96..36580839d42d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.module.scss @@ -18,7 +18,6 @@ .itemLabel { flex-grow: 1; - font-size: 16px; } .itemButton { diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 85f15053b4eb..0f6a46449748 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -7,6 +7,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useEffectOnce } from "react-use"; import * as yup from "yup"; +import Label from "components/Label"; import { Button } from "components/ui/Button"; import { Card } from "components/ui/Card"; import { InfoBox } from "components/ui/InfoBox"; @@ -108,7 +109,7 @@ export const InputsView: React.FC = () => {
      {inputs.value.map((input) => (
    1. -
      {input.definition.title || input.key}
      + -
      2. + ))} {inputs.value.map((input) => ( -
    2. - - -
      3. + ))}
    @@ -262,30 +235,26 @@ const InputModal = ({ label={formatMessage({ id: "connectorBuilder.inputModal.description" })} tooltip={formatMessage({ id: "connectorBuilder.inputModal.descriptionTooltip" })} /> - {values.type !== "unknown" ? ( + {values.type !== "unknown" && !isInferredInputOverride ? ( <> - {!isInferredInputOverride && ( - <> - { - setFieldValue("definition.default", undefined); - }} - label={formatMessage({ id: "connectorBuilder.inputModal.type" })} - tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} - /> - {values.type === "enum" && ( - - )} - + { + setFieldValue("definition.default", undefined); + }} + label={formatMessage({ id: "connectorBuilder.inputModal.type" })} + tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} + /> + {values.type === "enum" && ( + )} - {!isInferredInputOverride && ( - - )} + ) : ( - + {isInferredInputOverride ? ( + + ) : ( + + )} )} @@ -352,3 +323,30 @@ const InputModal = ({     ); }; + +const InputItem = ({ + input, + setInputInEditing, + isInferredInput, +}: { + input: BuilderFormInput; + setInputInEditing: (inputInEditing: InputInEditing) => void; + isInferredInput: boolean; +}): JSX.Element => { + return ( +
  • +
    {input.definition.title || input.key}
    + +
    • + ); +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 12d6b3fd5d89..d7a1bd39b008 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -703,6 +703,7 @@ "connectorBuilder.inputModal.enum": "Allowed values", "connectorBuilder.inputModal.enumTooltip": "The user will only be able to choose from one of these values. If none are provided the user will be able to enter any value", "connectorBuilder.inputModal.unsupportedInput": "Detailed configuration for this property type is disabled, switch to YAML view to edit", + "connectorBuilder.inputModal.inferredInputMessage": "Detailed configuration for this user input is disabled as it is tied to the selected authentication method", "connectorBuilder.key": "key", "connectorBuilder.value": "value", "connectorBuilder.addKeyValue": "Add", From eaec2791954d2f11b2406472d272b2c650f2213f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:11:03 +0100 Subject: [PATCH 086/100] unnecessary validation --- .../src/components/connectorBuilder/types.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index ea1af0428cff..c570dada2934 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -107,21 +107,6 @@ export const builderFormValidationSchema = yup.object().shape({ then: yup.string().required("form.empty.error"), otherwise: (schema) => schema.strip(), }), - api_token: yup.mixed().when("type", { - is: "ApiKeyAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - username: yup.mixed().when("type", { - is: "BasicHttpAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - password: yup.mixed().when("type", { - is: "BasicHttpAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), }), }), streams: yup.array().of( From a338590492d067c0f5fd07b157fe2c27be3c0fbb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:11:25 +0100 Subject: [PATCH 087/100] typo --- .../components/connectorBuilder/Builder/GlobalConfigView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 43c659f06797..97f2e75fe0e5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -22,7 +22,7 @@ export const GlobalConfigView: React.FC = () => { Date: Wed, 21 Dec 2022 12:14:40 +0100 Subject: [PATCH 088/100] unnecessary effect hook --- .../connectorBuilder/Builder/BuilderOneOf.tsx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx index 37f339071871..822d99927574 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx @@ -1,5 +1,5 @@ import { useField } from "formik"; -import React, { useEffect } from "react"; +import React from "react"; import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; @@ -28,12 +28,8 @@ interface BuilderOneOfProps { export const BuilderOneOf: React.FC = ({ options, path, label, tooltip }) => { const [, , oneOfPathHelpers] = useField(path); const typePath = `${path}.type`; - const [typePathField, , typePathHelpers] = useField(typePath); - const value = typePathField.value ?? options[0].typeValue; - - useEffect(() => { - typePathHelpers.setValue(value); - }, []); // eslint-disable-line react-hooks/exhaustive-deps + const [typePathField] = useField(typePath); + const value = typePathField.value; const selectedOption = options.find((option) => option.typeValue === value); From 6dd02d9ff3262e1de70fe8694603a3d58c6a0fde Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:39:51 +0100 Subject: [PATCH 089/100] build spec even for invalid forms but do not update stream list --- .../connectorBuilder/Builder/Builder.tsx | 4 +-- .../ConnectorBuilderStateService.tsx | 36 ++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index a9dbda62e6b4..06a2957f29ad 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -29,9 +29,7 @@ function getView(selectedView: BuilderView) { export const Builder: React.FC = ({ values, toggleYamlEditor }) => { const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); useEffect(() => { - if (builderFormValidationSchema.isValidSync(values)) { - setBuilderFormValues(values); - } + setBuilderFormValues(values, builderFormValidationSchema.isValidSync(values)); }, [values, setBuilderFormValues]); return ( diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b5b7b9818c2a..035b3ff9a23c 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -1,5 +1,5 @@ import { dump } from "js-yaml"; -import React, { useContext, useEffect, useMemo, useState } from "react"; +import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; @@ -44,7 +44,7 @@ interface Context { selectedView: BuilderView; configJson: StreamReadRequestBodyConfig; editorView: EditorView; - setBuilderFormValues: (values: BuilderFormValues) => void; + setBuilderFormValues: (values: BuilderFormValues, isInvalid: boolean) => void; setJsonManifest: (jsonValue: PatchedConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; @@ -60,10 +60,23 @@ export const ConnectorBuilderStateProvider: React.FC( + const [storedBuilderFormValues, setStoredBuilderFormValues] = useLocalStorage( "connectorBuilderFormValues", DEFAULT_BUILDER_FORM_VALUES ); + + const lastValidBuilderFormValuesRef = useRef(storedBuilderFormValues as BuilderFormValues); + + const setBuilderFormValues = useCallback( + (values: BuilderFormValues, isValid: boolean) => { + setStoredBuilderFormValues(values); + if (isValid) { + lastValidBuilderFormValuesRef.current = values; + } + }, + [setStoredBuilderFormValues] + ); + const builderFormValues = useMemo(() => { return { ...DEFAULT_BUILDER_FORM_VALUES, ...(storedBuilderFormValues ?? {}) }; }, [storedBuilderFormValues]); @@ -88,6 +101,21 @@ export const ConnectorBuilderStateProvider: React.FC("ui"); + const lastValidBuilderFormValues = lastValidBuilderFormValuesRef.current; + /** + * The json manifest derived from the last valid state of the builder form values. + * In the yaml view, this is undefined. Can still be invalid in case an invalid state is loaded from localstorage + */ + const lastValidJsonManifest = useMemo( + () => + editorView !== "ui" + ? undefined + : builderFormValues === lastValidBuilderFormValues + ? jsonManifest + : convertToManifest(lastValidBuilderFormValues), + [builderFormValues, editorView, jsonManifest, lastValidBuilderFormValues] + ); + // config const [configJson, setConfigJson] = useState({}); @@ -96,7 +124,7 @@ export const ConnectorBuilderStateProvider: React.FC Date: Wed, 21 Dec 2022 14:10:10 +0100 Subject: [PATCH 090/100] typos --- .../connectorBuilder/Builder/AuthenticationSection.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx index 74ac4e899f1f..af3835d7e34f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx @@ -79,7 +79,7 @@ export const AuthenticationSection: React.FC = () => { label="Token refresh endpoint" tooltip="The URL to call to obtain a new access token" /> - + @@ -88,7 +88,7 @@ export const AuthenticationSection: React.FC = () => { path="global.authenticator.scopes" optional label="Scopes" - tooltip="Scopes to rquest" + tooltip="Scopes to request" /> { type="string" path="global.authenticator.header" label="Header" - tooltip="Specific header of source for providing session token" + tooltip="Specific HTTP header of source API for providing session token" /> { type="string" path="global.authenticator.login_url" label="Login url" - tooltip="Url fot getting a specific session token" + tooltip="Url for getting a specific session token" /> Date: Wed, 21 Dec 2022 14:28:26 +0100 Subject: [PATCH 091/100] make sure validation error does not go away --- .../connectorBuilder/Builder/AuthenticationSection.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx index af3835d7e34f..89a06bcba7a6 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AuthenticationSection.tsx @@ -19,6 +19,7 @@ export const AuthenticationSection: React.FC = () => { typeValue: "ApiKeyAuthenticator", default: { api_token: "{{ config['api_key'] }}", + header: "", }, children: ( <> @@ -70,6 +71,7 @@ export const AuthenticationSection: React.FC = () => { client_secret: "{{ config['client_secret'] }}", refresh_token: "{{ config['client_refresh_token'] }}", refresh_request_body: [], + token_refresh_endpoint: "", }, children: ( <> From ed9860e4d526ad98121238cff9b782aa4af6f7a4 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 08:23:46 -0800 Subject: [PATCH 092/100] make primary key and cursor optional, and reorder --- .../connectorBuilder/Builder/StreamConfigView.tsx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 2282992f0577..a263d3dde156 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -82,23 +82,25 @@ export const StreamConfigView: React.FC = ({ streamNum }) label="HTTP Method" tooltip="HTTP method to use for requests sent to the API" /> + - From e029279edb459c2fc0dfff1458f991ab71df822e Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 10:46:41 -0800 Subject: [PATCH 093/100] save toggle group progress --- .../Builder/PaginationSection.tsx | 107 +++++++++++------- .../Builder/StreamConfigView.tsx | 7 -- .../Builder/ToggleGroupField.tsx | 40 +++++++ .../src/components/connectorBuilder/types.ts | 8 +- 4 files changed, 106 insertions(+), 56 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 672a0f817bfa..6ef7d7812cfe 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -1,12 +1,12 @@ import { useField } from "formik"; -import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; import { injectIntoValues } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; import { BuilderOneOf } from "./BuilderOneOf"; +import { ToggleGroupField } from "./ToggleGroupField"; interface PaginationSectionProps { streamFieldPath: (fieldPath: string) => string; @@ -21,12 +21,6 @@ export const PaginationSection: React.FC = ({ streamFiel strategy: { type: "OffsetIncrement", }, - pageSizeOption: { - inject_into: "request_parameter", - }, - pageTokenOption: { - inject_into: "request_parameter", - }, }); } else { helpers.setValue(undefined); @@ -35,13 +29,14 @@ export const PaginationSection: React.FC = ({ streamFiel const toggledOn = field.value !== undefined; const pageTokenOption = ( - - } + = ({ streamFiel tooltip="Configures which key should be used in the location that the page token is being injected into" optional /> - + ); - const pageSizeOption = ( - - } - > - - - - ); + // const pageTokenOption = ( + // + // } + // > + // + // + // + // ); + + // const pageSizeOption = ( + // + // } + // > + // + // + // + // ); return ( = ({ streamFiel tooltip="Size of pages to request from API" /> {pageTokenOption} - {pageSizeOption} + {/* {pageSizeOption} */} ), }, @@ -139,7 +160,7 @@ export const PaginationSection: React.FC = ({ streamFiel optional /> {pageTokenOption} - {pageSizeOption} + {/* {pageSizeOption} */} ), }, @@ -169,7 +190,7 @@ export const PaginationSection: React.FC = ({ streamFiel optional /> {pageTokenOption} - {pageSizeOption} + {/* {pageSizeOption} */} ), }, diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index a263d3dde156..09225da07c86 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -95,13 +95,6 @@ export const StreamConfigView: React.FC = ({ streamNum }) tooltip="Pointer into the response that should be used as the primary key when deduplicating records in the destination" optional /> - diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx new file mode 100644 index 000000000000..db2d7ab90d8d --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -0,0 +1,40 @@ +import { useField } from "formik"; + +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; +import { CheckBox } from "components/ui/CheckBox"; + +interface ToggleGroupFieldProps { + label: string; + tooltip: string; + fieldPath: string; + initialValues: unknown; +} + +export const ToggleGroupField: React.FC> = ({ + children, + label, + tooltip, + fieldPath, + initialValues, +}) => { + const [field, , helpers] = useField(fieldPath); + + return ( + + { + event.target.checked ? helpers.setValue(initialValues) : helpers.setValue(undefined); + }} + /> + +
    + } + > + {field.value !== undefined && children} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index a2f521d458eb..da1bd2c0cdee 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -50,7 +50,6 @@ export interface BuilderStream { urlPath: string; fieldPointer: string[]; primaryKey: string[]; - cursorField: string[]; httpMethod: "GET" | "POST"; requestOptions: { requestParameters: Array<[string, string]>; @@ -80,7 +79,6 @@ export const DEFAULT_BUILDER_STREAM_VALUES: BuilderStream = { urlPath: "", fieldPointer: [], primaryKey: [], - cursorField: [], httpMethod: "GET", requestOptions: { requestParameters: [], @@ -261,7 +259,6 @@ export const builderFormValidationSchema = yup.object().shape({ name: yup.string().required("form.empty.error"), urlPath: yup.string().required("form.empty.error"), fieldPointer: yup.array().of(yup.string()), - cursorField: yup.array().of(yup.string()), primaryKey: yup.array().of(yup.string()), httpMethod: yup.mixed().oneOf(["GET", "POST"]), requestOptions: yup.object().shape({ @@ -328,7 +325,6 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { name: stream.name, - stream_cursor_field: stream.cursorField, primary_key: stream.primaryKey, retriever: { name: stream.name, @@ -358,8 +354,8 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa page_token_option: { ...stream.paginator.pageTokenOption, // ensures that empty field_name is not set, as connector builder server cannot accept a field_name if inject_into is set to 'path' - field_name: stream.paginator.pageTokenOption.field_name - ? stream.paginator.pageTokenOption.field_name + field_name: stream.paginator.pageTokenOption?.field_name + ? stream.paginator.pageTokenOption?.field_name : undefined, }, page_size_option: stream.paginator.pageSizeOption, From e964eb885697f8d41d029b5b4a42ad4d1a4846d3 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 11:50:30 -0800 Subject: [PATCH 094/100] fix style of toggle label --- .../connectorBuilder/Builder/ToggleGroupField.module.scss | 6 ++++++ .../connectorBuilder/Builder/ToggleGroupField.tsx | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss new file mode 100644 index 000000000000..633d4fc7d84d --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss @@ -0,0 +1,6 @@ +@use "scss/variables"; + +.label { + display: flex; + gap: variables.$spacing-sm; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx index db2d7ab90d8d..2055ce7f2caa 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -4,6 +4,8 @@ import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; import { CheckBox } from "components/ui/CheckBox"; +import styles from "./ToggleGroupField.module.scss"; + interface ToggleGroupFieldProps { label: string; tooltip: string; @@ -23,7 +25,7 @@ export const ToggleGroupField: React.FC +
    { From a23ffb85c01a5834dd4f0296e61ea6649276a00e Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 11:51:06 -0800 Subject: [PATCH 095/100] handle empty values better --- .../src/components/connectorBuilder/Builder/BuilderField.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 1bffdfcc8d72..30171879af3f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -97,6 +97,9 @@ export const BuilderField: React.FC = ({ {...field} onChange={(e) => { field.onChange(e); + if (e.target.value === "") { + helpers.setValue(undefined); + } props.onChange?.(e.target.value); }} type={props.type} From e3c578d3e81292525ff6fc9b1f32bacbc7cc0493 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 14:12:57 -0800 Subject: [PATCH 096/100] fix page size/token option field validation and rendering --- .../Builder/InjectRequestOptionFields.tsx | 49 +++++++++ .../Builder/PaginationSection.tsx | 100 +++++------------- .../Builder/ToggleGroupField.module.scss | 8 +- .../Builder/ToggleGroupField.tsx | 29 +++-- .../src/components/connectorBuilder/types.ts | 26 +++-- 5 files changed, 113 insertions(+), 99 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx new file mode 100644 index 000000000000..7f556b44cc0e --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx @@ -0,0 +1,49 @@ +import { useField } from "formik"; + +import { RequestOption } from "core/request/ConnectorManifest"; + +import { injectIntoValues } from "../types"; +import { BuilderField } from "./BuilderField"; + +interface InjectRequestOptionFieldsProps { + path: string; + descriptor: string; + excludeInjectIntoValues?: string[]; +} + +export const InjectRequestOptionFields: React.FC = ({ + path, + descriptor, + excludeInjectIntoValues, +}) => { + const [field, , helpers] = useField(path); + + return ( + <> + !excludeInjectIntoValues.includes(val)) + : injectIntoValues + } + onChange={(newValue) => { + if (newValue === "path") { + helpers.setValue({ inject_into: newValue, field_name: undefined }); + } + }} + label="Inject into" + tooltip={`Configures where the ${descriptor} should be set on the HTTP requests`} + /> + {field.value.inject_into !== "path" && ( + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 6ef7d7812cfe..092e119725a2 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -1,11 +1,12 @@ import { useField } from "formik"; +import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; -import { injectIntoValues } from "../types"; import { BuilderCard } from "./BuilderCard"; import { BuilderField } from "./BuilderField"; import { BuilderOneOf } from "./BuilderOneOf"; +import { InjectRequestOptionFields } from "./InjectRequestOptionFields"; import { ToggleGroupField } from "./ToggleGroupField"; interface PaginationSectionProps { @@ -21,6 +22,9 @@ export const PaginationSection: React.FC = ({ streamFiel strategy: { type: "OffsetIncrement", }, + pageTokenOption: { + inject_into: "request_parameter", + }, }); } else { helpers.setValue(undefined); @@ -29,84 +33,36 @@ export const PaginationSection: React.FC = ({ streamFiel const toggledOn = field.value !== undefined; const pageTokenOption = ( + + } + > + + + ); + + const pageSizeOption = ( - - ); - // const pageTokenOption = ( - // - // } - // > - // - // - // - // ); - - // const pageSizeOption = ( - // - // } - // > - // - // - // - // ); - return ( = ({ streamFiel label="Page size" tooltip="Size of pages to request from API" /> + {pageSizeOption} {pageTokenOption} - {/* {pageSizeOption} */} ), }, @@ -159,8 +115,8 @@ export const PaginationSection: React.FC = ({ streamFiel tooltip="Page number to start requesting pages from" optional /> + {pageSizeOption} {pageTokenOption} - {/* {pageSizeOption} */} ), }, @@ -189,8 +145,8 @@ export const PaginationSection: React.FC = ({ streamFiel tooltip="Condition that determines when to stop requesting further pages" optional /> + {pageSizeOption} {pageTokenOption} - {/* {pageSizeOption} */} ), }, diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss index 633d4fc7d84d..1f016d2cb288 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss @@ -2,5 +2,11 @@ .label { display: flex; - gap: variables.$spacing-sm; + align-items: center; + gap: variables.$spacing-md; + height: 34px; + + label { + padding-bottom: 0; + } } diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx index 2055ce7f2caa..bd25d13ab793 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.tsx @@ -21,22 +21,19 @@ export const ToggleGroupField: React.FC { const [field, , helpers] = useField(fieldPath); + const enabled = field.value !== undefined; - return ( - - { - event.target.checked ? helpers.setValue(initialValues) : helpers.setValue(undefined); - }} - /> - -
    - } - > - {field.value !== undefined && children} - + const labelComponent = ( +
    + { + event.target.checked ? helpers.setValue(initialValues) : helpers.setValue(undefined); + }} + /> + +
    ); + + return enabled ? {children} : labelComponent; }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index da1bd2c0cdee..7701e9325d07 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -213,14 +213,6 @@ export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] } export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; -const requestOptionSchema = yup - .object() - .shape({ - inject_into: yup.mixed().oneOf(injectIntoValues), - field_name: yup.string(), - }) - .notRequired() - .default(undefined); export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ @@ -269,8 +261,22 @@ export const builderFormValidationSchema = yup.object().shape({ paginator: yup .object() .shape({ - pageSizeOption: requestOptionSchema, - pageTokenOption: requestOptionSchema, + pageSizeOption: yup + .object() + .shape({ + inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), + field_name: yup.string().required("form.empty.error"), + }) + .notRequired() + .default(undefined), + pageTokenOption: yup.object().shape({ + inject_into: yup.mixed().oneOf(injectIntoValues), + field_name: yup.mixed().when("inject_into", { + is: "path", + then: (schema) => schema.strip(), + otherwise: yup.string().required("form.empty.error"), + }), + }), strategy: yup .object({ page_size: yup.mixed().when("type", { From 24bfc32ea1f6ea70baf66202fe771615f5b55a3c Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 14:27:18 -0800 Subject: [PATCH 097/100] handle cursor pagination page size option correctly --- .../Builder/PaginationSection.tsx | 27 ++++++++++++------- .../src/components/connectorBuilder/types.ts | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 092e119725a2..4082e34cf04e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -15,6 +15,8 @@ interface PaginationSectionProps { export const PaginationSection: React.FC = ({ streamFieldPath }) => { const [field, , helpers] = useField(streamFieldPath("paginator")); + const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size")); + const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption")); const handleToggle = (newToggleValue: boolean) => { if (newToggleValue) { @@ -90,7 +92,7 @@ export const PaginationSection: React.FC = ({ streamFiel type="number" path={streamFieldPath("paginator.strategy.page_size")} label="Page size" - tooltip="Size of pages to request from API" + tooltip="Set the size of each page" /> {pageSizeOption} {pageTokenOption} @@ -106,7 +108,7 @@ export const PaginationSection: React.FC = ({ streamFiel type="number" path={streamFieldPath("paginator.strategy.page_size")} label="Page size" - tooltip="Size of pages to request from API" + tooltip="Set the size of each page" /> = ({ streamFiel tooltip="Value of the cursor to send in requests to the API" /> { + if (newValue === undefined || newValue === "") { + pageSizeOptionHelpers.setValue(undefined); + } + }} + label="Page size" + tooltip="Set the size of each page" optional /> - {pageSizeOption} + {pageSizeField.value && pageSizeField.value !== "" && pageSizeOption} {pageTokenOption} ), diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 7701e9325d07..f2b78f4ecc2e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -58,8 +58,8 @@ export interface BuilderStream { }; paginator?: { strategy: DefaultPaginatorAllOfPaginationStrategy; - pageSizeOption: RequestOption; pageTokenOption: RequestOption; + pageSizeOption?: RequestOption; }; } From 25eb39152c972af5f11f383ad68ed1f3f85c49e4 Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 21:49:17 -0800 Subject: [PATCH 098/100] save stream slicer progress --- .../Builder/StreamConfigView.tsx | 2 + .../Builder/StreamSlicerSection.tsx | 190 ++++++++++++++++++ .../src/components/connectorBuilder/types.ts | 3 + 3 files changed, 195 insertions(+) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 09225da07c86..1171aac969e4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -16,6 +16,7 @@ import { BuilderTitle } from "./BuilderTitle"; import { KeyValueListField } from "./KeyValueListField"; import { PaginationSection } from "./PaginationSection"; import styles from "./StreamConfigView.module.scss"; +import { StreamSlicerSection } from "./StreamSlicerSection"; interface StreamConfigViewProps { streamNum: number; @@ -97,6 +98,7 @@ export const StreamConfigView: React.FC = ({ streamNum }) /> + string; +} + +export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { + const [field, , helpers] = useField(streamFieldPath("paginator")); + + const handleToggle = (newToggleValue: boolean) => { + if (newToggleValue) { + helpers.setValue({ + type: "ListStreamSlicer", + slice_values: [], + cursor_field: "", + }); + } else { + helpers.setValue(undefined); + } + }; + const toggledOn = field.value !== undefined; + + return ( + + ), + toggledOn, + onToggle: handleToggle, + }} + > + + + + + + + ), + }, + { + label: "Datetime", + typeValue: "DatetimeStreamSlicer", + children: ( + <> + + + + + + + + + + } + > + + + + + } + > + + + + + + ), + }, + ]} + /> + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index f2b78f4ecc2e..bd8d7f5fef44 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -15,6 +15,7 @@ import { SessionTokenAuthenticator, DefaultPaginatorAllOfPaginationStrategy, RequestOption, + SimpleRetrieverAllOfStreamSlicer, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -61,6 +62,7 @@ export interface BuilderStream { pageTokenOption: RequestOption; pageSizeOption?: RequestOption; }; + streamSlicer?: SimpleRetrieverAllOfStreamSlicer; } export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { @@ -305,6 +307,7 @@ export const builderFormValidationSchema = yup.object().shape({ }) .notRequired() .default(undefined), + streamSlicer: yup.object().notRequired().default(undefined), }) ), }); From 3651ae2a8b208a05f495c8751c522e4c0feac9ff Mon Sep 17 00:00:00 2001 From: lmossman Date: Tue, 20 Dec 2022 22:22:57 -0800 Subject: [PATCH 099/100] finish stream slicer --- .../Builder/StreamSlicerSection.tsx | 9 +-- .../src/components/connectorBuilder/types.ts | 65 ++++++++++++++++++- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx index 537e8ab03384..b74aeab85830 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx @@ -14,7 +14,7 @@ interface StreamSlicerSectionProps { } export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { - const [field, , helpers] = useField(streamFieldPath("paginator")); + const [field, , helpers] = useField(streamFieldPath("streamSlicer")); const handleToggle = (newToggleValue: boolean) => { if (newToggleValue) { @@ -105,13 +105,6 @@ export const StreamSlicerSection: React.FC = ({ stream label="Step" tooltip="Time interval for which to break up stream into slices, e.g. 1d" /> - schema.strip(), + }), + request_option: yup.mixed().when("type", { + is: "ListStreamSlicer", + then: requestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + start_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + end_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + step: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + datetime_format: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + start_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: requestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + end_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: requestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + stream_state_field_start: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + stream_state_field_end: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + lookback_window: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }) + .notRequired() + .default(undefined), }) ), }); @@ -332,6 +393,7 @@ function builderFormAuthenticatorToAuthenticator( export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { + console.log("stream.streamSlicer", stream.streamSlicer); return { name: stream.name, primary_key: stream.primaryKey, @@ -372,6 +434,7 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa url_base: values.global?.urlBase, } : { type: "NoPagination" }, + stream_slicer: stream.streamSlicer, config: {}, }, config: {}, From 1e3cad9cd57704dfa62c645d5b9739d38d6697b1 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 21 Dec 2022 16:37:17 -0800 Subject: [PATCH 100/100] fix stream slicer fields and validation --- .../connectorBuilder/Builder/BuilderField.tsx | 7 +- .../Builder/PaginationSection.tsx | 2 +- .../Builder/StreamSlicerSection.tsx | 133 +++++++++--------- .../src/components/connectorBuilder/types.ts | 33 +++-- 4 files changed, 89 insertions(+), 86 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 30171879af3f..4b2443affc7b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -32,6 +32,7 @@ interface BaseFieldProps { tooltip?: string; readOnly?: boolean; optional?: boolean; + pattern?: RegExp; } type BuilderFieldProps = BaseFieldProps & @@ -66,6 +67,7 @@ export const BuilderField: React.FC = ({ tooltip, optional = false, readOnly, + pattern, ...props }) => { const [field, meta, helpers] = useField(path); @@ -116,7 +118,10 @@ export const BuilderField: React.FC = ({ )} {hasError && ( - + )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx index 4082e34cf04e..8d37011d29b9 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx @@ -50,7 +50,7 @@ export const PaginationSection: React.FC = ({ streamFiel const pageSizeOption = ( string; @@ -64,21 +65,21 @@ export const StreamSlicerSection: React.FC = ({ stream label="Cursor field" tooltip="Field on record to use as the cursor" /> - - + + + ), }, @@ -87,6 +88,12 @@ export const StreamSlicerSection: React.FC = ({ stream typeValue: "DatetimeStreamSlicer", children: ( <> + = ({ stream path={streamFieldPath("streamSlicer.step")} label="Step" tooltip="Time interval for which to break up stream into slices, e.g. 1d" + pattern={timeDeltaRegex} /> = ({ stream label="Cursor field" tooltip="Field on record to use as the cursor" /> - - - } + - - - - - } + + - - - + + + ), diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index d0e56dc450e1..9ca9795b8994 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -215,6 +215,17 @@ export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] } export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; +const nonPathRequestOptionSchema = yup + .object() + .shape({ + inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), + field_name: yup.string().required("form.empty.error"), + }) + .notRequired() + .default(undefined); + +// eslint-disable-next-line no-useless-escape +export const timeDeltaRegex = /^(([\.\d]+?)y)?(([\.\d]+?)m)?(([\.\d]+?)w)?(([\.\d]+?)d)?$/; export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ @@ -263,14 +274,7 @@ export const builderFormValidationSchema = yup.object().shape({ paginator: yup .object() .shape({ - pageSizeOption: yup - .object() - .shape({ - inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), - field_name: yup.string().required("form.empty.error"), - }) - .notRequired() - .default(undefined), + pageSizeOption: nonPathRequestOptionSchema, pageTokenOption: yup.object().shape({ inject_into: yup.mixed().oneOf(injectIntoValues), field_name: yup.mixed().when("inject_into", { @@ -316,11 +320,7 @@ export const builderFormValidationSchema = yup.object().shape({ then: yup.array().of(yup.string()), otherwise: (schema) => schema.strip(), }), - request_option: yup.mixed().when("type", { - is: "ListStreamSlicer", - then: requestOptionSchema, - otherwise: (schema) => schema.strip(), - }), + request_option: nonPathRequestOptionSchema, start_datetime: yup.mixed().when("type", { is: "DatetimeStreamSlicer", then: yup.string().required("form.empty.error"), @@ -333,7 +333,7 @@ export const builderFormValidationSchema = yup.object().shape({ }), step: yup.mixed().when("type", { is: "DatetimeStreamSlicer", - then: yup.string().required("form.empty.error"), + then: yup.string().matches(timeDeltaRegex, "form.pattern.error").required("form.empty.error"), otherwise: (schema) => schema.strip(), }), datetime_format: yup.mixed().when("type", { @@ -343,12 +343,12 @@ export const builderFormValidationSchema = yup.object().shape({ }), start_time_option: yup.mixed().when("type", { is: "DatetimeStreamSlicer", - then: requestOptionSchema, + then: nonPathRequestOptionSchema, otherwise: (schema) => schema.strip(), }), end_time_option: yup.mixed().when("type", { is: "DatetimeStreamSlicer", - then: requestOptionSchema, + then: nonPathRequestOptionSchema, otherwise: (schema) => schema.strip(), }), stream_state_field_start: yup.mixed().when("type", { @@ -393,7 +393,6 @@ function builderFormAuthenticatorToAuthenticator( export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { - console.log("stream.streamSlicer", stream.streamSlicer); return { name: stream.name, primary_key: stream.primaryKey,