From df816818f9a07b9ff47c89e92298b4c5b8387e77 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 1 Jun 2023 09:57:16 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=8E=89=20Connector=20builder:?= =?UTF-8?q?=20Form-encoded=20and=20freeform=20request=20bodies=20(#6854)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: flash1293 Co-authored-by: Lake Mossman --- .../connectorBuilder/Builder/BuilderField.tsx | 14 +++ .../Builder/RequestOptionSection.tsx | 87 +++++++++++++++++ .../Builder/StreamConfigView.tsx | 24 +---- .../convertManifestToBuilderForm.ts | 29 +++++- .../src/components/connectorBuilder/types.ts | 85 +++++++++++++---- .../useManifestToBuilderForm.test.ts | 93 +++++++++++++++++++ airbyte-webapp/src/locales/en.json | 2 +- 7 files changed, 291 insertions(+), 43 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index e06f3df7c80..75de71a337d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -6,6 +6,7 @@ import { FormattedMessage } from "react-intl"; import { ControlLabels } from "components/LabeledControl"; import { LabeledSwitch } from "components/LabeledSwitch"; +import { CodeEditor } from "components/ui/CodeEditor"; import { ComboBox, Option } from "components/ui/ComboBox"; import DatePicker from "components/ui/DatePicker"; import { DropDown } from "components/ui/DropDown"; @@ -61,6 +62,7 @@ export type BuilderFieldProps = BaseFieldProps & | { type: "boolean"; onChange?: (newValue: boolean) => void } | { type: "array"; onChange?: (newValue: string[]) => void; itemType?: string } | { type: "textarea"; onChange?: (newValue: string[]) => void } + | { type: "jsoneditor"; onChange?: (newValue: string[]) => void } | { type: "enum"; onChange?: (newValue: string) => void; options: string[] } | { type: "combobox"; onChange?: (newValue: string) => void; options: Option[] } ); @@ -200,6 +202,18 @@ export const BuilderField: React.FC = ({ onBlur={field.onBlur} /> )} + {props.type === "jsoneditor" && ( + { + setValue(val); + }} + /> + )} {props.type === "array" && (
(fieldPath: T) => `streams.${number}.${T}`; + currentStreamIndex: number; +} +export const RequestOptionSection: React.FC = ({ streamFieldPath, currentStreamIndex }) => { + const { formatMessage } = useIntl(); + + const bodyValue = useBuilderWatch(streamFieldPath("requestOptions.requestBody")); + + const getBodyOptions = (): OneOfOption[] => [ + { + label: "JSON (key-value pairs)", + typeValue: "json_list", + default: { + values: [], + }, + children: ( + + ), + }, + { + label: "Form encoded (key-value pairs)", + typeValue: "form_list", + default: { + values: [], + }, + children: ( + + ), + }, + { + label: "JSON (free form)", + typeValue: "json_freeform", + default: { + value: bodyValue.type === "json_list" ? JSON.stringify(Object.fromEntries(bodyValue.values)) : "{}", + }, + children: ( + + ), + }, + ]; + + return ( + + + + + + ); +}; + +RequestOptionSection.displayName = "RequestOptionSection"; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index f50bdf9ee33..01ec78e15a5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -28,10 +28,10 @@ import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs"; import { BuilderTitle } from "./BuilderTitle"; import { ErrorHandlerSection } from "./ErrorHandlerSection"; import { IncrementalSection } from "./IncrementalSection"; -import { KeyValueListField } from "./KeyValueListField"; import { getOptionsByManifest } from "./manifestHelpers"; import { PaginationSection } from "./PaginationSection"; import { PartitionSection } from "./PartitionSection"; +import { RequestOptionSection } from "./RequestOptionSection"; import styles from "./StreamConfigView.module.scss"; import { TransformationSection } from "./TransformationSection"; import { SchemaConflictIndicator } from "../SchemaConflictIndicator"; @@ -94,27 +94,7 @@ export const StreamConfigView: React.FC = React.memo(({ s - - - - - + ) : ( diff --git a/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts b/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts index 75735690377..62adcb666c7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts +++ b/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts @@ -164,8 +164,7 @@ const manifestStreamToBuilder = ( requestOptions: { requestParameters: Object.entries(requester.request_parameters ?? {}), requestHeaders: Object.entries(requester.request_headers ?? {}), - // try getting this from request_body_data first, and if not set then pull from request_body_json - requestBody: Object.entries(requester.request_body_data ?? requester.request_body_json ?? {}), + requestBody: requesterToRequestBody(stream.name, requester), }, primaryKey: manifestPrimaryKeyToBuilder(stream), paginator: manifestPaginatorToBuilder(retriever.paginator, stream.name), @@ -184,6 +183,32 @@ const manifestStreamToBuilder = ( }; }; +function requesterToRequestBody( + streamName: string | undefined, + requester: HttpRequester +): BuilderStream["requestOptions"]["requestBody"] { + if (requester.request_body_data && typeof requester.request_body_data === "object") { + return { type: "form_list", values: Object.entries(requester.request_body_data) }; + } + if (requester.request_body_data && typeof requester.request_body_data === "string") { + throw new ManifestCompatibilityError(streamName, "request_body_data is a string, but should be an object"); + } + if (!requester.request_body_json) { + return { type: "json_list", values: [] }; + } + const allStringValues = Object.values(requester.request_body_json).every((value) => typeof value === "string"); + if (allStringValues) { + return { type: "json_list", values: Object.entries(requester.request_body_json) }; + } + return { + type: "json_freeform", + value: + typeof requester.request_body_json === "string" + ? requester.request_body_json + : formatJson(requester.request_body_json), + }; +} + function manifestPartitionRouterToBuilder( partitionRouter: SimpleRetrieverPartitionRouter | SimpleRetrieverPartitionRouterAnyOfItem | undefined, serializedStreamToIndex: Record, diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 3cc313b0f72..3c66818beb3 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -154,6 +154,20 @@ export interface BuilderIncrementalSync export const INCREMENTAL_SYNC_USER_INPUT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"; +export type BuilderRequestBody = + | { + type: "json_list"; + values: Array<[string, string]>; + } + | { + type: "json_freeform"; + value: string; + } + | { + type: "form_list"; + values: Array<[string, string]>; + }; + export interface BuilderStream { id: string; name: string; @@ -164,7 +178,7 @@ export interface BuilderStream { requestOptions: { requestParameters: Array<[string, string]>; requestHeaders: Array<[string, string]>; - requestBody: Array<[string, string]>; + requestBody: BuilderRequestBody; }; paginator?: BuilderPaginator; transformations?: BuilderTransformation[]; @@ -245,7 +259,10 @@ export const DEFAULT_BUILDER_STREAM_VALUES: Omit = { requestOptions: { requestParameters: [], requestHeaders: [], - requestBody: [], + requestBody: { + type: "json_list", + values: [], + }, }, }; @@ -463,6 +480,21 @@ const keyValueListSchema = yup.array().of(yup.array().of(yup.string().required(" const yupNumberOrEmptyString = yup.number().transform((value) => (isNaN(value) ? undefined : value)); +const jsonString = yup.string().test({ + test: (val: string | undefined) => { + if (!val) { + return true; + } + try { + JSON.parse(val); + return true; + } catch { + return false; + } + }, + message: "connectorBuilder.invalidJSON", +}); + export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ connectorName: yup.string().required("form.empty.error").max(256, "connectorBuilder.maxLength"), @@ -498,22 +530,20 @@ export const builderFormValidationSchema = yup.object().shape({ requestOptions: yup.object().shape({ requestParameters: keyValueListSchema, requestHeaders: keyValueListSchema, - requestBody: keyValueListSchema, - }), - schema: yup.string().test({ - test: (val: string | undefined) => { - if (!val) { - return true; - } - try { - JSON.parse(val); - return true; - } catch { - return false; - } - }, - message: "connectorBuilder.invalidSchema", + requestBody: yup.object().shape({ + values: yup.mixed().when("type", { + is: (val: string) => val === "form_list" || val === "json_list", + then: keyValueListSchema, + otherwise: (schema) => schema.strip(), + }), + value: yup.mixed().when("type", { + is: (val: string) => val === "json_freeform", + then: jsonString, + otherwise: (schema) => schema.strip(), + }), + }), }), + schema: jsonString, paginator: yup .object() .shape({ @@ -909,6 +939,25 @@ function parseSchemaString(schema?: string): DeclarativeStreamSchemaLoader { } } +function builderRequestBodyToStreamRequestBody(stream: BuilderStream) { + try { + return { + request_body_json: + stream.requestOptions.requestBody.type === "json_list" + ? Object.fromEntries(stream.requestOptions.requestBody.values) + : stream.requestOptions.requestBody.type === "json_freeform" + ? JSON.parse(stream.requestOptions.requestBody.value) + : undefined, + request_body_data: + stream.requestOptions.requestBody.type === "form_list" + ? Object.fromEntries(stream.requestOptions.requestBody.values) + : undefined, + }; + } catch { + return {}; + } +} + function builderStreamToDeclarativeSteam( values: BuilderFormValues, stream: BuilderStream, @@ -928,9 +977,9 @@ function builderStreamToDeclarativeSteam( http_method: stream.httpMethod, request_parameters: Object.fromEntries(stream.requestOptions.requestParameters), request_headers: Object.fromEntries(stream.requestOptions.requestHeaders), - request_body_json: Object.fromEntries(stream.requestOptions.requestBody), authenticator: builderAuthenticatorToManifest(values.global), error_handler: buildCompositeErrorHandler(stream.errorHandler), + ...builderRequestBodyToStreamRequestBody(stream), }, record_selector: { type: "RecordSelector", diff --git a/airbyte-webapp/src/components/connectorBuilder/useManifestToBuilderForm.test.ts b/airbyte-webapp/src/components/connectorBuilder/useManifestToBuilderForm.test.ts index 27d903553ad..b144e956996 100644 --- a/airbyte-webapp/src/components/connectorBuilder/useManifestToBuilderForm.test.ts +++ b/airbyte-webapp/src/components/connectorBuilder/useManifestToBuilderForm.test.ts @@ -4,6 +4,7 @@ import { ConnectorManifest, DeclarativeStream } from "core/request/ConnectorMani import { DEFAULT_BUILDER_FORM_VALUES, DEFAULT_CONNECTOR_NAME, OLDEST_SUPPORTED_CDK_VERSION } from "./types"; import { convertToBuilderFormValues } from "./useManifestToBuilderForm"; +import { formatJson } from "./utils"; const baseManifest: ConnectorManifest = { type: "DeclarativeSource", @@ -267,6 +268,98 @@ describe("Conversion successfully results in", () => { ]); }); + it("request json body converted to key-value list", async () => { + const manifest: ConnectorManifest = { + ...baseManifest, + streams: [ + merge({}, stream1, { + retriever: { + requester: { + request_body_json: { + k1: "v1", + k2: "v2", + }, + }, + }, + }), + ], + }; + const formValues = await convertToBuilderFormValues(noOpResolve, manifest, DEFAULT_CONNECTOR_NAME); + expect(formValues.streams[0].requestOptions.requestBody).toEqual({ + type: "json_list", + values: [ + ["k1", "v1"], + ["k2", "v2"], + ], + }); + }); + + it("nested request json body converted to string", async () => { + const body = { + k1: { nested: "v1" }, + k2: "v2", + }; + const manifest: ConnectorManifest = { + ...baseManifest, + streams: [ + merge({}, stream1, { + retriever: { + requester: { + request_body_json: body, + }, + }, + }), + ], + }; + const formValues = await convertToBuilderFormValues(noOpResolve, manifest, DEFAULT_CONNECTOR_NAME); + expect(formValues.streams[0].requestOptions.requestBody).toEqual({ + type: "json_freeform", + value: formatJson(body), + }); + }); + + it("request data body converted to list", async () => { + const manifest: ConnectorManifest = { + ...baseManifest, + streams: [ + merge({}, stream1, { + retriever: { + requester: { + request_body_data: { + k1: "v1", + k2: "v2", + }, + }, + }, + }), + ], + }; + const formValues = await convertToBuilderFormValues(noOpResolve, manifest, DEFAULT_CONNECTOR_NAME); + expect(formValues.streams[0].requestOptions.requestBody).toEqual({ + type: "form_list", + values: [ + ["k1", "v1"], + ["k2", "v2"], + ], + }); + }); + + it("string body converted to string", async () => { + const manifest: ConnectorManifest = { + ...baseManifest, + streams: [ + merge({}, stream1, { + retriever: { + requester: { + request_body_data: "abc def", + }, + }, + }), + ], + }; + await expect(() => convertToBuilderFormValues(noOpResolve, manifest, DEFAULT_CONNECTOR_NAME)).rejects.toThrow(); + }); + it("primary key string converted to array", async () => { const manifest: ConnectorManifest = { ...baseManifest, diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 57193c252b4..7ebe16b6f74 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -847,7 +847,7 @@ "connectorBuilder.addNewPartitionRouter": "Add new partition router", "connectorBuilder.streamConfiguration": "Configuration", "connectorBuilder.streamSchema": "Declared schema", - "connectorBuilder.invalidSchema": "Invalid JSON - please fix schema to have it applied", + "connectorBuilder.invalidJSON": "Invalid JSON - please fix syntax to have it applied", "connectorBuilder.copyToPaginationTitle": "Copy pagination settings to...", "connectorBuilder.copyFromPaginationTitle": "Import pagination settings from...", "connectorBuilder.copyToPartitionRouterTitle": "Copy partitioning settings to...",