From 04d87cfe1a744152ba814d79665335961d5ac905 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 19 Dec 2022 17:14:43 +0100 Subject: [PATCH 01/11] allow auth configuration --- .../connectorBuilder/Builder/Builder.tsx | 6 +- .../connectorBuilder/Builder/BuilderOneOf.tsx | 66 ++++++++ .../Builder/BuilderSidebar.tsx | 7 +- .../Builder/GlobalConfigView.tsx | 66 ++++++++ .../connectorBuilder/Builder/InputsView.tsx | 147 +++++++++++------- .../Builder/UserInputField.module.scss | 12 ++ .../Builder/UserInputField.tsx | 18 +++ .../StreamTestingPanel/ConfigMenu.tsx | 5 +- .../src/components/connectorBuilder/types.ts | 98 +++++++++++- airbyte-webapp/src/locales/en.json | 1 + .../ConnectorBuilderPage.tsx | 63 ++++---- .../ConnectorBuilderStateService.tsx | 2 + 12 files changed, 397 insertions(+), 94 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UserInputField.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index 4107fe7331582..a9dbda62e6b4d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -3,7 +3,7 @@ import { useEffect } from "react"; import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; -import { BuilderFormValues } from "../types"; +import { builderFormValidationSchema, BuilderFormValues } from "../types"; import styles from "./Builder.module.scss"; import { BuilderSidebar } from "./BuilderSidebar"; import { GlobalConfigView } from "./GlobalConfigView"; @@ -29,7 +29,9 @@ function getView(selectedView: BuilderView) { export const Builder: React.FC = ({ values, toggleYamlEditor }) => { const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); useEffect(() => { - setBuilderFormValues(values); + if (builderFormValidationSchema.isValidSync(values)) { + setBuilderFormValues(values); + } }, [values, setBuilderFormValues]); return ( diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx new file mode 100644 index 0000000000000..37f3390718718 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx @@ -0,0 +1,66 @@ +import { useField } from "formik"; +import React, { useEffect } from "react"; + +import GroupControls from "components/GroupControls"; +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; + +interface Option { + label: string; + value: string; + default?: object; +} + +interface OneOfOption { + label: string; // label shown in the dropdown menu + typeValue: string; // value to set on the `type` field for this component - should match the oneOf type definition + default?: object; // default values for the path + children?: React.ReactNode; +} + +interface BuilderOneOfProps { + options: OneOfOption[]; + path: string; // path to the oneOf component in the json schema + label: string; + tooltip: string; +} + +export const BuilderOneOf: React.FC = ({ options, path, label, tooltip }) => { + const [, , oneOfPathHelpers] = useField(path); + const typePath = `${path}.type`; + const [typePathField, , typePathHelpers] = useField(typePath); + const value = typePathField.value ?? options[0].typeValue; + + useEffect(() => { + typePathHelpers.setValue(value); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + const selectedOption = options.find((option) => option.typeValue === value); + + return ( + } + dropdown={ + { + return { label: option.label, value: option.typeValue, default: option.default }; + })} + value={value ?? options[0].typeValue} + onChange={(selectedOption: Option) => { + if (selectedOption.value === value) { + return; + } + // clear all values for this oneOf and set selected option and default values + oneOfPathHelpers.setValue({ + type: selectedOption.value, + ...selectedOption.default, + }); + }} + /> + } + > + {selectedOption?.children} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx index 092b84374326b..3894595d67bd5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -17,7 +17,7 @@ import { } from "services/connectorBuilder/ConnectorBuilderStateService"; import { DownloadYamlButton } from "../DownloadYamlButton"; -import { BuilderFormValues } from "../types"; +import { BuilderFormValues, getInferredInputs } from "../types"; import { useBuilderErrors } from "../useBuilderErrors"; import { AddStreamButton } from "./AddStreamButton"; import styles from "./BuilderSidebar.module.scss"; @@ -115,7 +115,10 @@ export const BuilderSidebar: React.FC = ({ className, toggl onClick={() => handleViewSelect("inputs")} > - +
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 433d2d2e52960..43c659f067971 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -3,8 +3,10 @@ import { useIntl } from "react-intl"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; +import { BuilderOneOf } from "./BuilderOneOf"; import { BuilderTitle } from "./BuilderTitle"; import styles from "./GlobalConfigView.module.scss"; +import { UserInputField } from "./UserInputField"; export const GlobalConfigView: React.FC = () => { const { formatMessage } = useIntl(); @@ -16,6 +18,70 @@ export const GlobalConfigView: React.FC = () => { + + + + + + ), + }, + { + label: "Bearer", + typeValue: "BearerAuthenticator", + default: { + api_token: "{{ config['api_key'] }}", + }, + children: ( + + ), + }, + { + label: "Basic HTTP", + typeValue: "BasicHttpAuthenticator", + default: { + username: "{{ config['username'] }}", + password: "{{ config['password'] }}", + }, + children: ( + <> + + + + ), + }, + ]} + /> + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 0909e5774cfa9..755e7f0d7637e 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -14,7 +14,7 @@ import { Text } from "components/ui/Text"; import { FormikPatch } from "core/form/FormikPatch"; -import { BuilderFormInput } from "../types"; +import { BuilderFormInput, BuilderFormValues, getInferredInputs } from "../types"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; import styles from "./InputsView.module.scss"; @@ -28,6 +28,7 @@ interface InputInEditing { isNew?: boolean; showDefaultValueField: boolean; type: typeof supportedTypes[number]; + isInferredInputOverride: boolean; } function sluggify(str: string) { @@ -42,10 +43,14 @@ function newInputInEditing(): InputInEditing { isNew: true, showDefaultValueField: false, type: "string", + isInferredInputOverride: false, }; } -function formInputToInputInEditing({ key, definition, required }: BuilderFormInput): InputInEditing { +function formInputToInputInEditing( + { key, definition, required }: BuilderFormInput, + isInferredInputOverride: boolean +): InputInEditing { const supportedType = supportedTypes.find((type) => type === definition.type) || "unknown"; return { key, @@ -54,6 +59,7 @@ function formInputToInputInEditing({ key, definition, required }: BuilderFormInp isNew: false, showDefaultValueField: Boolean(definition.default), type: supportedType !== "unknown" && definition.enum ? "enum" : supportedType, + isInferredInputOverride, }; } @@ -77,6 +83,7 @@ function inputInEditingToFormInput({ export const InputsView: React.FC = () => { const { formatMessage } = useIntl(); + const { values, setFieldValue } = useFormikContext(); const [inputs, , helpers] = useField("inputs"); const [inputInEditing, setInputInEditing] = useState(undefined); const usedKeys = useMemo(() => inputs.value.map((input) => input.key), [inputs.value]); @@ -95,33 +102,52 @@ export const InputsView: React.FC = () => { }), [inputInEditing?.isNew, inputInEditing?.key, usedKeys] ); + const inferredInputs = getInferredInputs(values); + return ( - {inputs.value.length > 0 && ( - -
    - {inputs.value.map((input) => ( -
  1. -
    {input.definition.title || input.key}
    - -
    2. - ))} -
- - )} + {inputs.value.length > 0 || + (inferredInputs.length > 0 && ( + +
    + {inferredInputs.map((input) => ( +
  1. +
    {input.definition.title || input.key}
    + +
    2. + ))} + {inputs.value.map((input) => ( +
  2. +
    {input.definition.title || input.key}
    + +
    3. + ))} +
+ + ))} - - ))} - {inputs.value.map((input) => ( -
  • -
    {input.definition.title || input.key}
    - -
    • - ))} - - - ))} + {(inputs.value.length > 0 || inferredInputs.length > 0) && ( + +
      + {inferredInputs.map((input) => ( +
    1. +
      {input.definition.title || input.key}
      + +
      2. + ))} + {inputs.value.map((input) => ( +
    2. +
      {input.definition.title || input.key}
      + +
      3. + ))} +
    +     + )} - + ))} {inputs.value.map((input) => ( -
  • - - -
    • + ))} @@ -262,30 +235,26 @@ const InputModal = ({ label={formatMessage({ id: "connectorBuilder.inputModal.description" })} tooltip={formatMessage({ id: "connectorBuilder.inputModal.descriptionTooltip" })} /> - {values.type !== "unknown" ? ( + {values.type !== "unknown" && !isInferredInputOverride ? ( <> - {!isInferredInputOverride && ( - <> - { - setFieldValue("definition.default", undefined); - }} - label={formatMessage({ id: "connectorBuilder.inputModal.type" })} - tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} - /> - {values.type === "enum" && ( - - )} - + { + setFieldValue("definition.default", undefined); + }} + label={formatMessage({ id: "connectorBuilder.inputModal.type" })} + tooltip={formatMessage({ id: "connectorBuilder.inputModal.typeTooltip" })} + /> + {values.type === "enum" && ( + )} - {!isInferredInputOverride && ( - - )} + ) : ( - + {isInferredInputOverride ? ( + + ) : ( + + )} )} @@ -352,3 +323,30 @@ const InputModal = ({ ); }; + +const InputItem = ({ + input, + setInputInEditing, + isInferredInput, +}: { + input: BuilderFormInput; + setInputInEditing: (inputInEditing: InputInEditing) => void; + isInferredInput: boolean; +}): JSX.Element => { + return ( +
  • +
    {input.definition.title || input.key}
    + +
    • + ); +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 12d6b3fd5d89d..d7a1bd39b0083 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -703,6 +703,7 @@ "connectorBuilder.inputModal.enum": "Allowed values", "connectorBuilder.inputModal.enumTooltip": "The user will only be able to choose from one of these values. If none are provided the user will be able to enter any value", "connectorBuilder.inputModal.unsupportedInput": "Detailed configuration for this property type is disabled, switch to YAML view to edit", + "connectorBuilder.inputModal.inferredInputMessage": "Detailed configuration for this user input is disabled as it is tied to the selected authentication method", "connectorBuilder.key": "key", "connectorBuilder.value": "value", "connectorBuilder.addKeyValue": "Add", From eaec2791954d2f11b2406472d272b2c650f2213f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:11:03 +0100 Subject: [PATCH 05/11] unnecessary validation --- .../src/components/connectorBuilder/types.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index ea1af0428cff6..c570dada2934d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -107,21 +107,6 @@ export const builderFormValidationSchema = yup.object().shape({ then: yup.string().required("form.empty.error"), otherwise: (schema) => schema.strip(), }), - api_token: yup.mixed().when("type", { - is: "ApiKeyAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - username: yup.mixed().when("type", { - is: "BasicHttpAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), - password: yup.mixed().when("type", { - is: "BasicHttpAuthenticator", - then: yup.string().required("form.empty.error"), - otherwise: (schema) => schema.strip(), - }), }), }), streams: yup.array().of( From a338590492d067c0f5fd07b157fe2c27be3c0fbb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:11:25 +0100 Subject: [PATCH 06/11] typo --- .../components/connectorBuilder/Builder/GlobalConfigView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 43c659f067971..97f2e75fe0e5d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -22,7 +22,7 @@ export const GlobalConfigView: React.FC = () => { Date: Wed, 21 Dec 2022 12:14:40 +0100 Subject: [PATCH 07/11] unnecessary effect hook --- .../connectorBuilder/Builder/BuilderOneOf.tsx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx index 37f3390718718..822d999275748 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx @@ -1,5 +1,5 @@ import { useField } from "formik"; -import React, { useEffect } from "react"; +import React from "react"; import GroupControls from "components/GroupControls"; import { ControlLabels } from "components/LabeledControl"; @@ -28,12 +28,8 @@ interface BuilderOneOfProps { export const BuilderOneOf: React.FC = ({ options, path, label, tooltip }) => { const [, , oneOfPathHelpers] = useField(path); const typePath = `${path}.type`; - const [typePathField, , typePathHelpers] = useField(typePath); - const value = typePathField.value ?? options[0].typeValue; - - useEffect(() => { - typePathHelpers.setValue(value); - }, []); // eslint-disable-line react-hooks/exhaustive-deps + const [typePathField] = useField(typePath); + const value = typePathField.value; const selectedOption = options.find((option) => option.typeValue === value); From 6dd02d9ff3262e1de70fe8694603a3d58c6a0fde Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 21 Dec 2022 12:39:51 +0100 Subject: [PATCH 08/11] build spec even for invalid forms but do not update stream list --- .../connectorBuilder/Builder/Builder.tsx | 4 +-- .../ConnectorBuilderStateService.tsx | 36 ++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx index a9dbda62e6b4d..06a2957f29ade 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -29,9 +29,7 @@ function getView(selectedView: BuilderView) { export const Builder: React.FC = ({ values, toggleYamlEditor }) => { const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); useEffect(() => { - if (builderFormValidationSchema.isValidSync(values)) { - setBuilderFormValues(values); - } + setBuilderFormValues(values, builderFormValidationSchema.isValidSync(values)); }, [values, setBuilderFormValues]); return ( diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index b5b7b9818c2a6..035b3ff9a23cb 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -1,5 +1,5 @@ import { dump } from "js-yaml"; -import React, { useContext, useEffect, useMemo, useState } from "react"; +import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; @@ -44,7 +44,7 @@ interface Context { selectedView: BuilderView; configJson: StreamReadRequestBodyConfig; editorView: EditorView; - setBuilderFormValues: (values: BuilderFormValues) => void; + setBuilderFormValues: (values: BuilderFormValues, isInvalid: boolean) => void; setJsonManifest: (jsonValue: PatchedConnectorManifest) => void; setYamlEditorIsMounted: (value: boolean) => void; setYamlIsValid: (value: boolean) => void; @@ -60,10 +60,23 @@ export const ConnectorBuilderStateProvider: React.FC( + const [storedBuilderFormValues, setStoredBuilderFormValues] = useLocalStorage( "connectorBuilderFormValues", DEFAULT_BUILDER_FORM_VALUES ); + + const lastValidBuilderFormValuesRef = useRef(storedBuilderFormValues as BuilderFormValues); + + const setBuilderFormValues = useCallback( + (values: BuilderFormValues, isValid: boolean) => { + setStoredBuilderFormValues(values); + if (isValid) { + lastValidBuilderFormValuesRef.current = values; + } + }, + [setStoredBuilderFormValues] + ); + const builderFormValues = useMemo(() => { return { ...DEFAULT_BUILDER_FORM_VALUES, ...(storedBuilderFormValues ?? {}) }; }, [storedBuilderFormValues]); @@ -88,6 +101,21 @@ export const ConnectorBuilderStateProvider: React.FC("ui"); + const lastValidBuilderFormValues = lastValidBuilderFormValuesRef.current; + /** + * The json manifest derived from the last valid state of the builder form values. + * In the yaml view, this is undefined. Can still be invalid in case an invalid state is loaded from localstorage + */ + const lastValidJsonManifest = useMemo( + () => + editorView !== "ui" + ? undefined + : builderFormValues === lastValidBuilderFormValues + ? jsonManifest + : convertToManifest(lastValidBuilderFormValues), + [builderFormValues, editorView, jsonManifest, lastValidBuilderFormValues] + ); + // config const [configJson, setConfigJson] = useState({}); @@ -96,7 +124,7 @@ export const ConnectorBuilderStateProvider: React.FC Date: Wed, 21 Dec 2022 15:34:50 +0100 Subject: [PATCH 09/11] fix keys --- .../src/components/connectorBuilder/Builder/InputsView.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx index 3e72849dc2115..a934422b60f6f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx @@ -119,10 +119,10 @@ export const InputsView: React.FC = () => {
      {inferredInputs.map((input) => ( - + ))} {inputs.value.map((input) => ( - + ))}
    @@ -334,7 +334,7 @@ const InputItem = ({ isInferredInput: boolean; }): JSX.Element => { return ( -
  • +
  • {input.definition.title || input.key}
    + {isOpen &&
    {children}
    } +
    • + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx index 97f2e75fe0e5d..bc8e29a532a55 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -1,12 +1,11 @@ import { useIntl } from "react-intl"; +import { AuthenticationSection } from "./AuthenticationSection"; import { BuilderCard } from "./BuilderCard"; import { BuilderConfigView } from "./BuilderConfigView"; import { BuilderField } from "./BuilderField"; -import { BuilderOneOf } from "./BuilderOneOf"; import { BuilderTitle } from "./BuilderTitle"; import styles from "./GlobalConfigView.module.scss"; -import { UserInputField } from "./UserInputField"; export const GlobalConfigView: React.FC = () => { const { formatMessage } = useIntl(); @@ -18,70 +17,7 @@ export const GlobalConfigView: React.FC = () => { - - - - - - ), - }, - { - label: "Bearer", - typeValue: "BearerAuthenticator", - default: { - api_token: "{{ config['api_key'] }}", - }, - children: ( - - ), - }, - { - label: "Basic HTTP", - typeValue: "BasicHttpAuthenticator", - default: { - username: "{{ config['username'] }}", - password: "{{ config['password'] }}", - }, - children: ( - <> - - - - ), - }, - ]} - /> - + ); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index c570dada2934d..a76d49dec56ef 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -4,7 +4,16 @@ import * as yup from "yup"; import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; import { PatchedConnectorManifest } from "core/domain/connectorBuilder/PatchedConnectorManifest"; import { AirbyteJSONSchema } from "core/jsonSchema/types"; -import { DeclarativeStream, HttpRequesterAllOfAuthenticator } from "core/request/ConnectorManifest"; +import { + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + DeclarativeOauth2AuthenticatorAllOf, + DeclarativeStream, + HttpRequesterAllOfAuthenticator, + NoAuth, + SessionTokenAuthenticator, +} from "core/request/ConnectorManifest"; export interface BuilderFormInput { key: string; @@ -12,11 +21,22 @@ export interface BuilderFormInput { definition: AirbyteJSONSchema; } +type BuilderFormAuthenticator = ( + | NoAuth + | (Omit & { + refresh_request_body: Array<[string, string]>; + }) + | ApiKeyAuthenticator + | BearerAuthenticator + | BasicHttpAuthenticator + | SessionTokenAuthenticator +) & { type: string }; + export interface BuilderFormValues { global: { connectorName: string; urlBase: string; - authenticator: HttpRequesterAllOfAuthenticator; + authenticator: BuilderFormAuthenticator; }; inputs: BuilderFormInput[]; inferredInputOverrides: Record>; @@ -82,6 +102,68 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] { }, ]; } + if (values.global.authenticator.type === "OAuthAuthenticator") { + return [ + { + key: "client_id", + required: true, + definition: { + type: "string", + title: "Client ID", + airbyte_secret: true, + }, + }, + { + key: "client_secret", + required: true, + definition: { + type: "string", + title: "Client secret", + airbyte_secret: true, + }, + }, + { + key: "refresh_token", + required: true, + definition: { + type: "string", + title: "Refresh token", + airbyte_secret: true, + }, + }, + ]; + } + if (values.global.authenticator.type === "SessionTokenAuthenticator") { + return [ + { + key: "username", + required: false, + definition: { + type: "string", + title: "Username", + }, + }, + { + key: "password", + required: false, + definition: { + type: "string", + title: "Password", + airbyte_secret: true, + }, + }, + { + key: "session_token", + required: false, + definition: { + type: "string", + title: "Session token", + description: "Session token generated by user (if provided username and password are not required)", + airbyte_secret: true, + }, + }, + ]; + } return []; } @@ -103,7 +185,27 @@ export const builderFormValidationSchema = yup.object().shape({ urlBase: yup.string().required("form.empty.error"), authenticator: yup.object({ header: yup.mixed().when("type", { - is: "ApiKeyAuthenticator", + is: (type: string) => type === "ApiKeyAuthenticator" || type === "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + token_refresh_endpoint: yup.mixed().when("type", { + is: "OAuthAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + session_token_response_key: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + login_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + validate_session_url: yup.mixed().when("type", { + is: "SessionTokenAuthenticator", then: yup.string().required("form.empty.error"), otherwise: (schema) => schema.strip(), }), @@ -124,6 +226,24 @@ export const builderFormValidationSchema = yup.object().shape({ ), }); +function builderFormAuthenticatorToAuthenticator( + globalSettings: BuilderFormValues["global"] +): HttpRequesterAllOfAuthenticator { + if (globalSettings.authenticator.type === "OAuthAuthenticator") { + return { + ...globalSettings.authenticator, + refresh_request_body: Object.fromEntries(globalSettings.authenticator.refresh_request_body), + }; + } + if (globalSettings.authenticator.type === "SessionTokenAuthenticator") { + return { + ...globalSettings.authenticator, + api_url: globalSettings.urlBase, + }; + } + return globalSettings.authenticator as HttpRequesterAllOfAuthenticator; +} + export const convertToManifest = (values: BuilderFormValues): PatchedConnectorManifest => { const manifestStreams: DeclarativeStream[] = values.streams.map((stream) => { return { @@ -139,7 +259,7 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), request_body_data: Object.fromEntries(stream.requestOptions.requestBody), }, - authenticator: values.global.authenticator, + authenticator: builderFormAuthenticatorToAuthenticator(values.global), // TODO: remove these empty "config" values once they are no longer required in the connector manifest JSON schema config: {}, }, diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index d7a1bd39b0083..c56f066425df9 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -717,6 +717,7 @@ "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel", + "connectorBuilder.optionalFieldsLabel": "Optional fields", "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times", "jobs.noAttemptsFailure": "Failed to start job.", From 30c60b86d3dae430139bab60c9ed481660b5e81f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 22 Dec 2022 11:29:49 +0100 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=8E=89=20Connector=20bu?= =?UTF-8?q?ilder:=20Always=20validate=20inputs=20form=20(#20664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * validate user input outside of form * review comments --- .../StreamTestingPanel/ConfigMenu.module.scss | 6 +++ .../StreamTestingPanel/ConfigMenu.tsx | 31 ++++++++----- .../StreamTestingPanel/StreamTestButton.tsx | 14 +++++- .../StreamTestingPanel/StreamTester.tsx | 11 ++++- .../StreamTestingPanel.module.scss | 2 - .../StreamTestingPanel/StreamTestingPanel.tsx | 43 +++++++++++++++++-- airbyte-webapp/src/locales/en.json | 1 + 7 files changed, 87 insertions(+), 21 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss index bf4573ee6edbb..73b43de5ec13c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.module.scss @@ -33,3 +33,9 @@ gap: variables.$spacing-md; align-items: center; } + +.inputsErrorBadge { + position: absolute; + top: -1 * variables.$spacing-md; + right: -1 * variables.$spacing-md; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx index d2560c251ec7e..b2b3f224d96b3 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/ConfigMenu.tsx @@ -1,12 +1,12 @@ -import { faClose, faGear } from "@fortawesome/free-solid-svg-icons"; +import { faClose, faUser } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useLocalStorage } from "react-use"; import { Button } from "components/ui/Button"; import { InfoBox } from "components/ui/InfoBox"; import { Modal, ModalBody } from "components/ui/Modal"; +import { NumberBadge } from "components/ui/NumberBadge"; import { Tooltip } from "components/ui/Tooltip"; import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; @@ -18,10 +18,12 @@ import { ConfigMenuErrorBoundaryComponent } from "./ConfigMenuErrorBoundary"; interface ConfigMenuProps { className?: string; + configJsonErrors: number; + isOpen: boolean; + setIsOpen: (open: boolean) => void; } -export const ConfigMenu: React.FC = ({ className }) => { - const [isOpen, setIsOpen] = useState(false); +export const ConfigMenu: React.FC = ({ className, configJsonErrors, isOpen, setIsOpen }) => { const { formatMessage } = useIntl(); const { configJson, setConfigJson, jsonManifest, editorView, setEditorView } = useConnectorBuilderState(); @@ -36,13 +38,20 @@ export const ConfigMenu: React.FC = ({ className }) => { <> setIsOpen(true)} - disabled={!jsonManifest.spec} - icon={} - /> + <> + + {configJsonErrors > 0 && ( + + )} + } placement={editorView === "yaml" ? "left" : "top"} containerClassName={className} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx index 482893c176e02..4e382aef75184 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestButton.tsx @@ -14,13 +14,23 @@ import styles from "./StreamTestButton.module.scss"; interface StreamTestButtonProps { readStream: () => void; + hasConfigJsonErrors: boolean; + setTestInputOpen: (open: boolean) => void; } -export const StreamTestButton: React.FC = ({ readStream }) => { +export const StreamTestButton: React.FC = ({ + readStream, + hasConfigJsonErrors, + setTestInputOpen, +}) => { const { editorView, yamlIsValid, testStreamIndex } = useConnectorBuilderState(); const { hasErrors, validateAndTouch } = useBuilderErrors(); const handleClick = () => { + if (hasConfigJsonErrors) { + setTestInputOpen(true); + return; + } if (editorView === "yaml") { readStream(); return; @@ -39,7 +49,7 @@ export const StreamTestButton: React.FC = ({ readStream } tooltipContent = ; } - if (editorView === "ui" && hasErrors(true, ["global", testStreamIndex])) { + if ((editorView === "ui" && hasErrors(true, ["global", testStreamIndex])) || hasConfigJsonErrors) { showWarningIcon = true; tooltipContent = ; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 9c0908cd3ac14..dc8027a6d0a97 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -13,7 +13,10 @@ import { ResultDisplay } from "./ResultDisplay"; import { StreamTestButton } from "./StreamTestButton"; import styles from "./StreamTester.module.scss"; -export const StreamTester: React.FC = () => { +export const StreamTester: React.FC<{ + hasConfigJsonErrors: boolean; + setTestInputOpen: (open: boolean) => void; +}> = ({ hasConfigJsonErrors, setTestInputOpen }) => { const { formatMessage } = useIntl(); const { jsonManifest, configJson, streams, testStreamIndex } = useConnectorBuilderState(); const { @@ -55,7 +58,11 @@ export const StreamTester: React.FC = () => { {streams[testStreamIndex]?.url} - + {isFetching && (
    diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss index 3b1d67fa022d6..fa2474e5efbfd 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.module.scss @@ -15,8 +15,6 @@ $buttonHeight: 36px; position: absolute; top: variables.$spacing-lg; left: variables.$spacing-lg; - height: $buttonHeight; - width: $buttonHeight; } .streamSelector { diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx index 6cc2632bf6ad7..f3799ed1fe468 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTestingPanel.tsx @@ -1,10 +1,15 @@ -import React from "react"; +import React, { useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; +import { ValidationError } from "yup"; import { Heading } from "components/ui/Heading"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; +import { SourceDefinitionSpecificationDraft } from "core/domain/connector"; +import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; +import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; +import { StreamReadRequestBodyConfig } from "core/request/ConnectorBuilderClient"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import { links } from "utils/links"; @@ -13,8 +18,33 @@ import { StreamSelector } from "./StreamSelector"; import { StreamTester } from "./StreamTester"; import styles from "./StreamTestingPanel.module.scss"; +const EMPTY_SCHEMA = {}; + +function useConfigJsonErrors( + configJson: StreamReadRequestBodyConfig, + spec?: SourceDefinitionSpecificationDraft +): number { + return useMemo(() => { + try { + const jsonSchema = spec && spec.connectionSpecification ? spec.connectionSpecification : EMPTY_SCHEMA; + const formFields = jsonSchemaToFormBlock(jsonSchema); + const validationSchema = buildYupFormForJsonSchema(jsonSchema, formFields); + validationSchema.validateSync(configJson, { abortEarly: false }); + return 0; + } catch (e) { + if (ValidationError.isError(e)) { + return e.errors.length; + } + return 1; + } + }, [configJson, spec]); +} + export const StreamTestingPanel: React.FC = () => { - const { jsonManifest, streamListErrorMessage, yamlEditorIsMounted } = useConnectorBuilderState(); + const [isTestInputOpen, setTestInputOpen] = useState(false); + const { jsonManifest, configJson, streamListErrorMessage, yamlEditorIsMounted } = useConnectorBuilderState(); + + const configJsonErrors = useConfigJsonErrors(configJson, jsonManifest.spec); if (!yamlEditorIsMounted) { return ( @@ -38,10 +68,15 @@ export const StreamTestingPanel: React.FC = () => { )} {hasStreams && streamListErrorMessage === undefined && ( <> - +
    - + 0} setTestInputOpen={setTestInputOpen} />
    )} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index c56f066425df9..c405e8f56b28f 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -717,6 +717,7 @@ "connectorBuilder.inputsNoSpecUITooltip": "Add User Input fields to allow setting test inputs", "connectorBuilder.inputsNoSpecYAMLTooltip": "Add a spec to your manifest to allow setting test inputs", "connectorBuilder.setInUserInput": "This setting is configured as part of the user inputs in the testing panel", + "connectorBuilder.inputsButton": "Inputs", "connectorBuilder.optionalFieldsLabel": "Optional fields", "connectorBuilder.duplicateFieldID": "Make sure no field ID is used multiple times",