🪟🎉 Connector builder: Form-encoded and freeform request bodies (#6854)

Co-authored-by: flash1293 <flash1293@users.noreply.github.com>
Co-authored-by: Lake Mossman <lake@airbyte.io>

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx

@@ -6,6 +6,7 @@ import { FormattedMessage } from "react-intl";
 
 import { ControlLabels } from "components/LabeledControl";
 import { LabeledSwitch } from "components/LabeledSwitch";
+import { CodeEditor } from "components/ui/CodeEditor";
 import { ComboBox, Option } from "components/ui/ComboBox";
 import DatePicker from "components/ui/DatePicker";
 import { DropDown } from "components/ui/DropDown";
@@ -61,6 +62,7 @@ export type BuilderFieldProps = BaseFieldProps &
     | { type: "boolean"; onChange?: (newValue: boolean) => void }
     | { type: "array"; onChange?: (newValue: string[]) => void; itemType?: string }
     | { type: "textarea"; onChange?: (newValue: string[]) => void }
+    | { type: "jsoneditor"; onChange?: (newValue: string[]) => void }
     | { type: "enum"; onChange?: (newValue: string) => void; options: string[] }
     | { type: "combobox"; onChange?: (newValue: string) => void; options: Option[] }
   );
@@ -200,6 +202,18 @@ export const BuilderField: React.FC<BuilderFieldProps> = ({
           onBlur={field.onBlur}
         />
       )}
+      {props.type === "jsoneditor" && (
+        <CodeEditor
+          height="300px"
+          key={path}
+          value={field.value || ""}
+          language="json"
+          theme="airbyte-light"
+          onChange={(val: string | undefined) => {
+            setValue(val);
+          }}
+        />
+      )}
       {props.type === "array" && (
         <div data-testid={`tag-input-${path}`}>
           <ArrayField

airbyte-webapp/src/components/connectorBuilder/Builder/RequestOptionSection.tsx

@@ -0,0 +1,87 @@
+import { useIntl } from "react-intl";
+
+import { BuilderCard } from "./BuilderCard";
+import { BuilderField } from "./BuilderField";
+import { BuilderOneOf, OneOfOption } from "./BuilderOneOf";
+import { KeyValueListField } from "./KeyValueListField";
+import { useBuilderWatch } from "../types";
+
+interface RequestOptionSectionProps {
+  streamFieldPath: <T extends string>(fieldPath: T) => `streams.${number}.${T}`;
+  currentStreamIndex: number;
+}
+export const RequestOptionSection: React.FC<RequestOptionSectionProps> = ({ streamFieldPath, currentStreamIndex }) => {
+  const { formatMessage } = useIntl();
+
+  const bodyValue = useBuilderWatch(streamFieldPath("requestOptions.requestBody"));
+
+  const getBodyOptions = (): OneOfOption[] => [
+    {
+      label: "JSON (key-value pairs)",
+      typeValue: "json_list",
+      default: {
+        values: [],
+      },
+      children: (
+        <KeyValueListField
+          path={streamFieldPath("requestOptions.requestBody.values")}
+          manifestPath="HttpRequester.properties.request_body_json"
+        />
+      ),
+    },
+    {
+      label: "Form encoded (key-value pairs)",
+      typeValue: "form_list",
+      default: {
+        values: [],
+      },
+      children: (
+        <KeyValueListField
+          path={streamFieldPath("requestOptions.requestBody.values")}
+          manifestPath="HttpRequester.properties.request_body_data"
+        />
+      ),
+    },
+    {
+      label: "JSON (free form)",
+      typeValue: "json_freeform",
+      default: {
+        value: bodyValue.type === "json_list" ? JSON.stringify(Object.fromEntries(bodyValue.values)) : "{}",
+      },
+      children: (
+        <BuilderField
+          type="jsoneditor"
+          path={streamFieldPath("requestOptions.requestBody.value")}
+          manifestPath="HttpRequester.properties.request_body_json"
+        />
+      ),
+    },
+  ];
+
+  return (
+    <BuilderCard
+      copyConfig={{
+        path: "requestOptions",
+        currentStreamIndex,
+        copyFromLabel: formatMessage({ id: "connectorBuilder.copyFromRequestOptionsTitle" }),
+        copyToLabel: formatMessage({ id: "connectorBuilder.copyToRequestOptionsTitle" }),
+      }}
+    >
+      <KeyValueListField
+        path={streamFieldPath("requestOptions.requestParameters")}
+        manifestPath="HttpRequester.properties.request_parameters"
+      />
+      <KeyValueListField
+        path={streamFieldPath("requestOptions.requestHeaders")}
+        manifestPath="HttpRequester.properties.request_headers"
+      />
+      <BuilderOneOf
+        path={streamFieldPath("requestOptions.requestBody")}
+        label="Request body"
+        options={getBodyOptions()}
+      />
+    </BuilderCard>
+  );
+};
+
+RequestOptionSection.displayName = "RequestOptionSection";

airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx

@@ -28,10 +28,10 @@ import { BuilderFieldWithInputs } from "./BuilderFieldWithInputs";
 import { BuilderTitle } from "./BuilderTitle";
 import { ErrorHandlerSection } from "./ErrorHandlerSection";
 import { IncrementalSection } from "./IncrementalSection";
-import { KeyValueListField } from "./KeyValueListField";
 import { getOptionsByManifest } from "./manifestHelpers";
 import { PaginationSection } from "./PaginationSection";
 import { PartitionSection } from "./PartitionSection";
+import { RequestOptionSection } from "./RequestOptionSection";
 import styles from "./StreamConfigView.module.scss";
 import { TransformationSection } from "./TransformationSection";
 import { SchemaConflictIndicator } from "../SchemaConflictIndicator";
@@ -94,27 +94,7 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = React.memo(({ s
           <PartitionSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
           <ErrorHandlerSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
           <TransformationSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
-          <BuilderCard
-            copyConfig={{
-              path: "requestOptions",
-              currentStreamIndex: streamNum,
-              copyFromLabel: formatMessage({ id: "connectorBuilder.copyFromRequestOptionsTitle" }),
-              copyToLabel: formatMessage({ id: "connectorBuilder.copyToRequestOptionsTitle" }),
-            }}
-          >
-            <KeyValueListField
-              path={streamFieldPath("requestOptions.requestParameters")}
-              manifestPath="HttpRequester.properties.request_parameters"
-            />
-            <KeyValueListField
-              path={streamFieldPath("requestOptions.requestHeaders")}
-              manifestPath="HttpRequester.properties.request_headers"
-            />
-            <KeyValueListField
-              path={streamFieldPath("requestOptions.requestBody")}
-              manifestPath="HttpRequester.properties.request_body_json"
-            />
-          </BuilderCard>
+          <RequestOptionSection streamFieldPath={streamFieldPath} currentStreamIndex={streamNum} />
         </>
       ) : (
         <BuilderCard className={styles.schemaEditor}>

airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts

@@ -164,8 +164,7 @@ const manifestStreamToBuilder = (
     requestOptions: {
       requestParameters: Object.entries(requester.request_parameters ?? {}),
       requestHeaders: Object.entries(requester.request_headers ?? {}),
-      // try getting this from request_body_data first, and if not set then pull from request_body_json
-      requestBody: Object.entries(requester.request_body_data ?? requester.request_body_json ?? {}),
+      requestBody: requesterToRequestBody(stream.name, requester),
     },
     primaryKey: manifestPrimaryKeyToBuilder(stream),
     paginator: manifestPaginatorToBuilder(retriever.paginator, stream.name),
@@ -184,6 +183,32 @@ const manifestStreamToBuilder = (
   };
 };
 
+function requesterToRequestBody(
+  streamName: string | undefined,
+  requester: HttpRequester
+): BuilderStream["requestOptions"]["requestBody"] {
+  if (requester.request_body_data && typeof requester.request_body_data === "object") {
+    return { type: "form_list", values: Object.entries(requester.request_body_data) };
+  }
+  if (requester.request_body_data && typeof requester.request_body_data === "string") {
+    throw new ManifestCompatibilityError(streamName, "request_body_data is a string, but should be an object");
+  }
+  if (!requester.request_body_json) {
+    return { type: "json_list", values: [] };
+  }
+  const allStringValues = Object.values(requester.request_body_json).every((value) => typeof value === "string");
+  if (allStringValues) {
+    return { type: "json_list", values: Object.entries(requester.request_body_json) };
+  }
+  return {
+    type: "json_freeform",
+    value:
+      typeof requester.request_body_json === "string"
+        ? requester.request_body_json
+        : formatJson(requester.request_body_json),
+  };
+}
+
 function manifestPartitionRouterToBuilder(
   partitionRouter: SimpleRetrieverPartitionRouter | SimpleRetrieverPartitionRouterAnyOfItem | undefined,
   serializedStreamToIndex: Record<string, number>,

airbyte-webapp/src/components/connectorBuilder/types.ts

@@ -154,6 +154,20 @@ export interface BuilderIncrementalSync
 
 export const INCREMENTAL_SYNC_USER_INPUT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ";
 
+export type BuilderRequestBody =
+  | {
+      type: "json_list";
+      values: Array<[string, string]>;
+    }
+  | {
+      type: "json_freeform";
+      value: string;
+    }
+  | {
+      type: "form_list";
+      values: Array<[string, string]>;
+    };
+
 export interface BuilderStream {
   id: string;
   name: string;
@@ -164,7 +178,7 @@ export interface BuilderStream {
   requestOptions: {
     requestParameters: Array<[string, string]>;
     requestHeaders: Array<[string, string]>;
-    requestBody: Array<[string, string]>;
+    requestBody: BuilderRequestBody;
   };
   paginator?: BuilderPaginator;
   transformations?: BuilderTransformation[];
@@ -245,7 +259,10 @@ export const DEFAULT_BUILDER_STREAM_VALUES: Omit<BuilderStream, "id"> = {
   requestOptions: {
     requestParameters: [],
     requestHeaders: [],
-    requestBody: [],
+    requestBody: {
+      type: "json_list",
+      values: [],
+    },
   },
 };
 
@@ -463,6 +480,21 @@ const keyValueListSchema = yup.array().of(yup.array().of(yup.string().required("
 
 const yupNumberOrEmptyString = yup.number().transform((value) => (isNaN(value) ? undefined : value));
 
+const jsonString = yup.string().test({
+  test: (val: string | undefined) => {
+    if (!val) {
+      return true;
+    }
+    try {
+      JSON.parse(val);
+      return true;
+    } catch {
+      return false;
+    }
+  },
+  message: "connectorBuilder.invalidJSON",
+});
+
 export const builderFormValidationSchema = yup.object().shape({
   global: yup.object().shape({
     connectorName: yup.string().required("form.empty.error").max(256, "connectorBuilder.maxLength"),
@@ -498,22 +530,20 @@ export const builderFormValidationSchema = yup.object().shape({
         requestOptions: yup.object().shape({
           requestParameters: keyValueListSchema,
           requestHeaders: keyValueListSchema,
-          requestBody: keyValueListSchema,
-        }),
-        schema: yup.string().test({
-          test: (val: string | undefined) => {
-            if (!val) {
-              return true;
-            }
-            try {
-              JSON.parse(val);
-              return true;
-            } catch {
-              return false;
-            }
-          },
-          message: "connectorBuilder.invalidSchema",
+          requestBody: yup.object().shape({
+            values: yup.mixed().when("type", {
+              is: (val: string) => val === "form_list" || val === "json_list",
+              then: keyValueListSchema,
+              otherwise: (schema) => schema.strip(),
+            }),
+            value: yup.mixed().when("type", {
+              is: (val: string) => val === "json_freeform",
+              then: jsonString,
+              otherwise: (schema) => schema.strip(),
+            }),
+          }),
         }),
+        schema: jsonString,
         paginator: yup
           .object()
           .shape({
@@ -909,6 +939,25 @@ function parseSchemaString(schema?: string): DeclarativeStreamSchemaLoader {
   }
 }
 
+function builderRequestBodyToStreamRequestBody(stream: BuilderStream) {
+  try {
+    return {
+      request_body_json:
+        stream.requestOptions.requestBody.type === "json_list"
+          ? Object.fromEntries(stream.requestOptions.requestBody.values)
+          : stream.requestOptions.requestBody.type === "json_freeform"
+          ? JSON.parse(stream.requestOptions.requestBody.value)
+          : undefined,
+      request_body_data:
+        stream.requestOptions.requestBody.type === "form_list"
+          ? Object.fromEntries(stream.requestOptions.requestBody.values)
+          : undefined,
+    };
+  } catch {
+    return {};
+  }
+}
+
 function builderStreamToDeclarativeSteam(
   values: BuilderFormValues,
   stream: BuilderStream,
@@ -928,9 +977,9 @@ function builderStreamToDeclarativeSteam(
         http_method: stream.httpMethod,
         request_parameters: Object.fromEntries(stream.requestOptions.requestParameters),
         request_headers: Object.fromEntries(stream.requestOptions.requestHeaders),
-        request_body_json: Object.fromEntries(stream.requestOptions.requestBody),
         authenticator: builderAuthenticatorToManifest(values.global),
         error_handler: buildCompositeErrorHandler(stream.errorHandler),
+        ...builderRequestBodyToStreamRequestBody(stream),
       },
       record_selector: {
         type: "RecordSelector",