diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx
index 55e21b7ee07d7..bdbdf4fe2edf6 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx
@@ -1,4 +1,4 @@
-import { useField } from "formik";
+import { FastField, FastFieldProps, FieldInputProps } from "formik";
 import { ReactNode } from "react";
 import { FormattedMessage } from "react-intl";
 
@@ -64,24 +64,26 @@ const ArrayField: React.FC<ArrayFieldProps> = ({ name, value, setValue, error })
   return <TagInput name={name} fieldValue={value} onChange={(value) => setValue(value)} error={error} />;
 };
 
-export const BuilderField: React.FC<BuilderFieldProps> = ({
+const InnerBuilderField: React.FC<BuilderFieldProps & FastFieldProps<unknown>> = ({
   path,
   label,
   tooltip,
   optional = false,
   readOnly,
   pattern,
+  field,
+  meta,
+  form,
   adornment,
   ...props
 }) => {
-  const [field, meta, helpers] = useField(path);
   const hasError = !!meta.error && meta.touched;
 
   if (props.type === "boolean") {
     return (
       <LabeledSwitch
-        {...field}
-        checked={field.value}
+        {...(field as FieldInputProps<string>)}
+        checked={field.value as boolean}
         label={
           <>
             {label} {tooltip && <InfoTooltip placement="top-start">{tooltip}</InfoTooltip>}
@@ -93,7 +95,7 @@ export const BuilderField: React.FC<BuilderFieldProps> = ({
 
   const setValue = (newValue: unknown) => {
     props.onChange?.(newValue as string & string[]);
-    helpers.setValue(newValue);
+    form.setFieldValue(path, newValue);
   };
 
   return (
@@ -104,23 +106,28 @@ export const BuilderField: React.FC<BuilderFieldProps> = ({
           onChange={(e) => {
             field.onChange(e);
             if (e.target.value === "") {
-              helpers.setValue(undefined);
+              form.setFieldValue(path, undefined);
             }
             props.onChange?.(e.target.value);
           }}
           className={props.className}
           type={props.type}
-          value={field.value ?? ""}
+          value={(field.value as string | number | undefined) ?? ""}
           error={hasError}
           readOnly={readOnly}
           adornment={adornment}
         />
       )}
       {props.type === "array" && (
-        <ArrayField name={path} value={field.value ?? []} setValue={setValue} error={hasError} />
+        <ArrayField
+          name={path}
+          value={(field.value as string[] | undefined) ?? []}
+          setValue={setValue}
+          error={hasError}
+        />
       )}
       {props.type === "enum" && (
-        <EnumField options={props.options} value={field.value} setValue={setValue} error={hasError} />
+        <EnumField options={props.options} value={field.value as string} setValue={setValue} error={hasError} />
       )}
       {hasError && (
         <Text className={styles.error}>
@@ -133,3 +140,13 @@ export const BuilderField: React.FC<BuilderFieldProps> = ({
     </ControlLabels>
   );
 };
+
+export const BuilderField: React.FC<BuilderFieldProps> = (props) => {
+  return (
+    <FastField name={props.path}>
+      {({ field, form, meta }: FastFieldProps<unknown>) => (
+        <InnerBuilderField {...props} field={field} form={form} meta={meta} />
+      )}
+    </FastField>
+  );
+};
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderFieldWithInputs.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderFieldWithInputs.tsx
index 0751eda7e6988..9e48547c4722c 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderFieldWithInputs.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderFieldWithInputs.tsx
@@ -2,6 +2,7 @@ import { faPlus, faUser } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { useField } from "formik";
 import { useMemo, useState } from "react";
+import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 
 import { ListBox, ListBoxControlButtonProps, Option } from "components/ui/ListBox";
@@ -26,20 +27,18 @@ export const BuilderFieldWithInputs: React.FC<BuilderFieldProps> = (props) => {
   );
 };
 
-export const UserInputHelper = ({
-  setValue,
-  currentValue,
-}: {
+interface UserInputHelperProps {
   setValue: (value: string) => void;
   currentValue: string;
-}) => {
+}
+
+export const UserInputHelper = (props: UserInputHelperProps) => {
   const { formatMessage } = useIntl();
-  const [modalOpen, setModalOpen] = useState(false);
   const { builderFormValues } = useConnectorBuilderFormState();
   const listOptions = useMemo(() => {
     const options: Array<Option<string | undefined>> = [
       ...builderFormValues.inputs,
-      ...getInferredInputs(builderFormValues),
+      ...getInferredInputs(builderFormValues.global, builderFormValues.inferredInputOverrides),
     ].map((input) => ({
       label: input.definition.title || input.key,
       value: input.key,
@@ -50,45 +49,56 @@ export const UserInputHelper = ({
       icon: <FontAwesomeIcon icon={faPlus} />,
     });
     return options;
-  }, [builderFormValues, formatMessage]);
-  return (
-    <>
-      <ListBox<string | undefined>
-        buttonClassName={styles.button}
-        optionClassName={styles.option}
-        className={styles.container}
-        selectedOptionClassName={styles.selectedOption}
-        controlButton={UserInputHelperControlButton}
-        selectedValue={undefined}
-        onSelect={(selectedValue) => {
-          if (selectedValue) {
-            setValue(`${currentValue || ""}{{ config['${selectedValue}'] }}`);
-          } else {
-            // This hack is necessary because listbox will put the focus back when the option list gets hidden, which conflicts with the auto-focus setting of the modal.
-            // As it's not possible to prevent listbox from forcing the focus back on the button component, this will wait until the focus went to the button, then opens the modal
-            // so it can move it to the first input
-            setTimeout(() => {
-              setModalOpen(true);
-            }, 50);
-          }
-        }}
-        options={listOptions}
-      />
-      {modalOpen && (
-        <InputForm
-          inputInEditing={newInputInEditing()}
-          onClose={(newInput) => {
-            setModalOpen(false);
-            if (!newInput) {
-              return;
+  }, [builderFormValues.global, builderFormValues.inferredInputOverrides, builderFormValues.inputs, formatMessage]);
+  return <InnerUserInputHelper {...props} listOptions={listOptions} />;
+};
+
+const InnerUserInputHelper = React.memo(
+  ({
+    setValue,
+    currentValue,
+    listOptions,
+  }: UserInputHelperProps & { listOptions: Array<Option<string | undefined>> }) => {
+    const [modalOpen, setModalOpen] = useState(false);
+    return (
+      <>
+        <ListBox<string | undefined>
+          buttonClassName={styles.button}
+          optionClassName={styles.option}
+          className={styles.container}
+          selectedOptionClassName={styles.selectedOption}
+          controlButton={UserInputHelperControlButton}
+          selectedValue={undefined}
+          onSelect={(selectedValue) => {
+            if (selectedValue) {
+              setValue(`${currentValue || ""}{{ config['${selectedValue}'] }}`);
+            } else {
+              // This hack is necessary because listbox will put the focus back when the option list gets hidden, which conflicts with the auto-focus setting of the modal.
+              // As it's not possible to prevent listbox from forcing the focus back on the button component, this will wait until the focus went to the button, then opens the modal
+              // so it can move it to the first input
+              setTimeout(() => {
+                setModalOpen(true);
+              }, 50);
             }
-            setValue(`${currentValue}{{ config['${newInput.key}'] }}`);
           }}
+          options={listOptions}
         />
-      )}
-    </>
-  );
-};
+        {modalOpen && (
+          <InputForm
+            inputInEditing={newInputInEditing()}
+            onClose={(newInput) => {
+              setModalOpen(false);
+              if (!newInput) {
+                return;
+              }
+              setValue(`${currentValue}{{ config['${newInput.key}'] }}`);
+            }}
+          />
+        )}
+      </>
+    );
+  }
+);
 
 const UserInputHelperControlButton: React.FC<ListBoxControlButtonProps<string | undefined>> = () => {
   return (
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx
index e6bec6bbbbcdb..d8046b0e4c974 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderOneOf.tsx
@@ -1,4 +1,4 @@
-import { useField } from "formik";
+import { FastField, FastFieldProps } from "formik";
 import React from "react";
 
 import GroupControls from "components/GroupControls";
@@ -25,10 +25,14 @@ interface BuilderOneOfProps {
   tooltip: string;
 }
 
-export const BuilderOneOf: React.FC<BuilderOneOfProps> = ({ options, path, label, tooltip }) => {
-  const [, , oneOfPathHelpers] = useField(path);
-  const typePath = `${path}.type`;
-  const [typePathField] = useField(typePath);
+const InnerBuilderOneOf: React.FC<BuilderOneOfProps & FastFieldProps<string>> = ({
+  options,
+  label,
+  tooltip,
+  field: typePathField,
+  path,
+  form,
+}) => {
   const value = typePathField.value;
 
   const selectedOption = options.find((option) => option.typeValue === value);
@@ -48,7 +52,7 @@ export const BuilderOneOf: React.FC<BuilderOneOfProps> = ({ options, path, label
               return;
             }
             // clear all values for this oneOf and set selected option and default values
-            oneOfPathHelpers.setValue({
+            form.setFieldValue(path, {
               type: selectedOption.value,
               ...selectedOption.default,
             });
@@ -60,3 +64,10 @@ export const BuilderOneOf: React.FC<BuilderOneOfProps> = ({ options, path, label
     </GroupControls>
   );
 };
+export const BuilderOneOf: React.FC<BuilderOneOfProps> = (props) => {
+  return (
+    <FastField name={`${props.path}.type`}>
+      {(fastFieldProps: FastFieldProps<string>) => <InnerBuilderOneOf {...props} {...fastFieldProps} />}
+    </FastField>
+  );
+};
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx
index d8197ce8ff98b..b4ddabeb56a55 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx
@@ -111,7 +111,9 @@ export const BuilderSidebar: React.FC<BuilderSidebarProps> = React.memo(({ class
         <FontAwesomeIcon icon={faUser} />
         <FormattedMessage
           id="connectorBuilder.userInputs"
-          values={{ number: values.inputs.length + getInferredInputs(values).length }}
+          values={{
+            number: values.inputs.length + getInferredInputs(values.global, values.inferredInputOverrides).length,
+          }}
         />
       </ViewSelectButton>
 
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsForm.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsForm.tsx
index f01b0e93dc597..69395a462283c 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsForm.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsForm.tsx
@@ -70,7 +70,10 @@ export const InputForm = ({
 }) => {
   const { values, setFieldValue } = useFormikContext<BuilderFormValues>();
   const [inputs, , helpers] = useField<BuilderFormInput[]>("inputs");
-  const inferredInputs = useMemo(() => getInferredInputs(values), [values]);
+  const inferredInputs = useMemo(
+    () => getInferredInputs(values.global, values.inferredInputOverrides),
+    [values.global, values.inferredInputOverrides]
+  );
   const usedKeys = useMemo(
     () => [...inputs.value, ...inferredInputs].map((input) => input.key),
     [inputs.value, inferredInputs]
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx
index 293350c5c264f..dd97ef6a0a5e7 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx
@@ -19,7 +19,10 @@ export const InputsView: React.FC = () => {
   const { formatMessage } = useIntl();
   const { values } = useFormikContext<BuilderFormValues>();
   const [inputInEditing, setInputInEditing] = useState<InputInEditing | undefined>(undefined);
-  const inferredInputs = useMemo(() => getInferredInputs(values), [values]);
+  const inferredInputs = useMemo(
+    () => getInferredInputs(values.global, values.inferredInputOverrides),
+    [values.global, values.inferredInputOverrides]
+  );
 
   return (
     <BuilderConfigView heading={formatMessage({ id: "connectorBuilder.inputsTitle" })}>
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx
index c425c59427725..da8a621a9a9a3 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/KeyValueListField.tsx
@@ -1,4 +1,5 @@
 import { useField } from "formik";
+import React, { useRef } from "react";
 import { FormattedMessage } from "react-intl";
 
 import GroupControls from "components/GroupControls";
@@ -60,29 +61,53 @@ interface KeyValueListFieldProps {
 export const KeyValueListField: React.FC<KeyValueListFieldProps> = ({ path, label, tooltip }) => {
   const [{ value: keyValueList }, , { setValue: setKeyValueList }] = useField<Array<[string, string]>>(path);
 
+  // need to wrap the setter into a ref because it will be a new function on every formik state update
+  const setKeyValueListRef = useRef(setKeyValueList);
+  setKeyValueListRef.current = setKeyValueList;
+
   return (
-    <GroupControls
-      label={<ControlLabels label={label} infoTooltipContent={tooltip} />}
-      control={
-        <Button type="button" variant="secondary" onClick={() => setKeyValueList([...keyValueList, ["", ""]])}>
-          <FormattedMessage id="connectorBuilder.addKeyValue" />
-        </Button>
-      }
-    >
-      {keyValueList.map((keyValue, keyValueIndex) => (
-        <KeyValueInput
-          key={keyValueIndex}
-          keyValue={keyValue}
-          onChange={(newKeyValue) => {
-            const updatedList = keyValueList.map((entry, index) => (index === keyValueIndex ? newKeyValue : entry));
-            setKeyValueList(updatedList);
-          }}
-          onRemove={() => {
-            const updatedList = keyValueList.filter((_, index) => index !== keyValueIndex);
-            setKeyValueList(updatedList);
-          }}
-        />
-      ))}
-    </GroupControls>
+    <KeyValueList label={label} tooltip={tooltip} keyValueList={keyValueList} setKeyValueList={setKeyValueListRef} />
   );
 };
+
+const KeyValueList = React.memo(
+  ({
+    keyValueList,
+    setKeyValueList,
+    label,
+    tooltip,
+  }: Omit<KeyValueListFieldProps, "path"> & {
+    keyValueList: Array<[string, string]>;
+    setKeyValueList: React.MutableRefObject<(val: Array<[string, string]>) => void>;
+  }) => {
+    return (
+      <GroupControls
+        label={<ControlLabels label={label} infoTooltipContent={tooltip} />}
+        control={
+          <Button
+            type="button"
+            variant="secondary"
+            onClick={() => setKeyValueList.current([...keyValueList, ["", ""]])}
+          >
+            <FormattedMessage id="connectorBuilder.addKeyValue" />
+          </Button>
+        }
+      >
+        {keyValueList.map((keyValue, keyValueIndex) => (
+          <KeyValueInput
+            key={keyValueIndex}
+            keyValue={keyValue}
+            onChange={(newKeyValue) => {
+              const updatedList = keyValueList.map((entry, index) => (index === keyValueIndex ? newKeyValue : entry));
+              setKeyValueList.current(updatedList);
+            }}
+            onRemove={() => {
+              const updatedList = keyValueList.filter((_, index) => index !== keyValueIndex);
+              setKeyValueList.current(updatedList);
+            }}
+          />
+        ))}
+      </GroupControls>
+    );
+  }
+);
diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx
index 6fd028f3cb8a3..1209de8654718 100644
--- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx
+++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx
@@ -3,6 +3,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import classNames from "classnames";
 import { useField } from "formik";
 import { useState } from "react";
+import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
 
 import Indicator from "components/Indicator";
@@ -29,35 +30,13 @@ interface StreamConfigViewProps {
   hasMultipleStreams: boolean;
 }
 
-export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, hasMultipleStreams }) => {
+export const StreamConfigView: React.FC<StreamConfigViewProps> = React.memo(({ streamNum, hasMultipleStreams }) => {
   const { formatMessage } = useIntl();
-  const [field, , helpers] = useField<BuilderStream[]>("streams");
-  const [selectedTab, setSelectedTab] = useState<"configuration" | "schema">("configuration");
-  const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
-  const { setSelectedView } = useConnectorBuilderFormState();
 
+  const [selectedTab, setSelectedTab] = useState<"configuration" | "schema">("configuration");
   const streamPath = `streams[${streamNum}]`;
   const streamFieldPath = (fieldPath: string) => `${streamPath}.${fieldPath}`;
 
-  const handleDelete = () => {
-    openConfirmationModal({
-      text: "connectorBuilder.deleteStreamModal.text",
-      title: "connectorBuilder.deleteStreamModal.title",
-      submitButtonText: "connectorBuilder.deleteStreamModal.submitButton",
-      onSubmit: () => {
-        const updatedStreams = field.value.filter((_, index) => index !== streamNum);
-        const streamToSelect = streamNum >= updatedStreams.length ? updatedStreams.length - 1 : streamNum;
-        const viewToSelect: BuilderView = updatedStreams.length === 0 ? "global" : streamToSelect;
-        helpers.setValue(updatedStreams);
-        setSelectedView(viewToSelect);
-        closeConfirmationModal();
-      },
-    });
-  };
-
-  const [, meta] = useField<string | undefined>(streamFieldPath("schema"));
-  const hasSchemaErrors = Boolean(meta.error);
-
   return (
     <BuilderConfigView
       heading={formatMessage({ id: "connectorBuilder.stream" })}
@@ -65,33 +44,12 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, h
     >
       {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */}
       <BuilderTitle path={streamFieldPath("name")} label="Stream Name" size="md" />
-      <div className={styles.controls}>
-        <StreamTab
-          label={formatMessage({ id: "connectorBuilder.streamConfiguration" })}
-          selected={selectedTab === "configuration"}
-          onSelect={() => setSelectedTab("configuration")}
-        />
-        <StreamTab
-          label={formatMessage({ id: "connectorBuilder.streamSchema" })}
-          selected={selectedTab === "schema"}
-          onSelect={() => setSelectedTab("schema")}
-          showErrorIndicator={hasSchemaErrors}
-        />
-        <AddStreamButton
-          onAddStream={(addedStreamNum) => {
-            setSelectedView(addedStreamNum);
-          }}
-          initialValues={field.value[streamNum]}
-          button={
-            <button className={styles.controlButton} type="button">
-              <FontAwesomeIcon icon={faCopy} />
-            </button>
-          }
-        />
-        <button className={classNames(styles.deleteButton, styles.controlButton)} type="button" onClick={handleDelete}>
-          <FontAwesomeIcon icon={faTrashCan} />
-        </button>
-      </div>
+      <StreamControls
+        streamNum={streamNum}
+        selectedTab={selectedTab}
+        setSelectedTab={setSelectedTab}
+        streamFieldPath={streamFieldPath}
+      />
       {selectedTab === "configuration" ? (
         <>
           <BuilderCard>
@@ -149,6 +107,70 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum, h
       )}
     </BuilderConfigView>
   );
+});
+
+const StreamControls = ({
+  streamNum,
+  selectedTab,
+  setSelectedTab,
+  streamFieldPath,
+}: {
+  streamNum: number;
+  streamFieldPath: (path: string) => string;
+  setSelectedTab: (tab: "configuration" | "schema") => void;
+  selectedTab: "configuration" | "schema";
+}) => {
+  const { formatMessage } = useIntl();
+  const [field, , helpers] = useField<BuilderStream[]>("streams");
+  const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService();
+  const { setSelectedView } = useConnectorBuilderFormState();
+  const [, meta] = useField<string | undefined>(streamFieldPath("schema"));
+  const hasSchemaErrors = Boolean(meta.error);
+
+  const handleDelete = () => {
+    openConfirmationModal({
+      text: "connectorBuilder.deleteStreamModal.text",
+      title: "connectorBuilder.deleteStreamModal.title",
+      submitButtonText: "connectorBuilder.deleteStreamModal.submitButton",
+      onSubmit: () => {
+        const updatedStreams = field.value.filter((_, index) => index !== streamNum);
+        const streamToSelect = streamNum >= updatedStreams.length ? updatedStreams.length - 1 : streamNum;
+        const viewToSelect: BuilderView = updatedStreams.length === 0 ? "global" : streamToSelect;
+        helpers.setValue(updatedStreams);
+        setSelectedView(viewToSelect);
+        closeConfirmationModal();
+      },
+    });
+  };
+  return (
+    <div className={styles.controls}>
+      <StreamTab
+        label={formatMessage({ id: "connectorBuilder.streamConfiguration" })}
+        selected={selectedTab === "configuration"}
+        onSelect={() => setSelectedTab("configuration")}
+      />
+      <StreamTab
+        label={formatMessage({ id: "connectorBuilder.streamSchema" })}
+        selected={selectedTab === "schema"}
+        onSelect={() => setSelectedTab("schema")}
+        showErrorIndicator={hasSchemaErrors}
+      />
+      <AddStreamButton
+        onAddStream={(addedStreamNum) => {
+          setSelectedView(addedStreamNum);
+        }}
+        initialValues={field.value[streamNum]}
+        button={
+          <button className={styles.controlButton} type="button">
+            <FontAwesomeIcon icon={faCopy} />
+          </button>
+        }
+      />
+      <button className={classNames(styles.deleteButton, styles.controlButton)} type="button" onClick={handleDelete}>
+        <FontAwesomeIcon icon={faTrashCan} />
+      </button>
+    </div>
+  );
 };
 
 const StreamTab = ({
diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts
index 6f60eac5cfcd9..b7d5e827422b3 100644
--- a/airbyte-webapp/src/components/connectorBuilder/types.ts
+++ b/airbyte-webapp/src/components/connectorBuilder/types.ts
@@ -115,8 +115,8 @@ export const DEFAULT_BUILDER_STREAM_VALUES: Omit<BuilderStream, "id"> = {
   },
 };
 
-function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
-  if (values.global.authenticator.type === "ApiKeyAuthenticator") {
+function getInferredInputList(global: BuilderFormValues["global"]): BuilderFormInput[] {
+  if (global.authenticator.type === "ApiKeyAuthenticator") {
     return [
       {
         key: "api_key",
@@ -129,7 +129,7 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
       },
     ];
   }
-  if (values.global.authenticator.type === "BearerAuthenticator") {
+  if (global.authenticator.type === "BearerAuthenticator") {
     return [
       {
         key: "api_key",
@@ -142,7 +142,7 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
       },
     ];
   }
-  if (values.global.authenticator.type === "BasicHttpAuthenticator") {
+  if (global.authenticator.type === "BasicHttpAuthenticator") {
     return [
       {
         key: "username",
@@ -163,7 +163,7 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
       },
     ];
   }
-  if (values.global.authenticator.type === "OAuthAuthenticator") {
+  if (global.authenticator.type === "OAuthAuthenticator") {
     return [
       {
         key: "client_id",
@@ -194,7 +194,7 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
       },
     ];
   }
-  if (values.global.authenticator.type === "SessionTokenAuthenticator") {
+  if (global.authenticator.type === "SessionTokenAuthenticator") {
     return [
       {
         key: "username",
@@ -228,13 +228,16 @@ function getInferredInputList(values: BuilderFormValues): BuilderFormInput[] {
   return [];
 }
 
-export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] {
-  const inferredInputs = getInferredInputList(values);
+export function getInferredInputs(
+  global: BuilderFormValues["global"],
+  inferredInputOverrides: BuilderFormValues["inferredInputOverrides"]
+): BuilderFormInput[] {
+  const inferredInputs = getInferredInputList(global);
   return inferredInputs.map((input) =>
-    values.inferredInputOverrides[input.key]
+    inferredInputOverrides[input.key]
       ? {
           ...input,
-          definition: { ...input.definition, ...values.inferredInputOverrides[input.key] },
+          definition: { ...input.definition, ...inferredInputOverrides[input.key] },
         }
       : input
   );
@@ -577,7 +580,7 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest
     builderStreamToDeclarativeSteam(values, stream, [])
   );
 
-  const allInputs = [...values.inputs, ...getInferredInputs(values)];
+  const allInputs = [...values.inputs, ...getInferredInputs(values.global, values.inferredInputOverrides)];
 
   const specSchema: JSONSchema7 = {
     $schema: "http://json-schema.org/draft-07/schema#",
diff --git a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
index be7c38afacedb..e1413607c6a07 100644
--- a/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
+++ b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx
@@ -29,16 +29,25 @@ const ConnectorBuilderPageInner: React.FC = React.memo(() => {
   const initialFormValues = useRef(builderFormValues);
   return useMemo(
     () => (
-      <Formik initialValues={initialFormValues.current} onSubmit={noop} validationSchema={builderFormValidationSchema}>
-        {({ values, validateForm }) => (
-          <Panels
-            editorView={editorView}
-            validateForm={validateForm}
-            switchToUI={switchToUI}
-            values={values}
-            switchToYaml={switchToYaml}
-          />
-        )}
+      <Formik
+        initialValues={initialFormValues.current}
+        validateOnBlur={false}
+        validateOnChange={false}
+        validateOnMount={false}
+        onSubmit={noop}
+        validationSchema={builderFormValidationSchema}
+      >
+        {(props) => {
+          return (
+            <Panels
+              editorView={editorView}
+              validateForm={props.validateForm}
+              switchToUI={switchToUI}
+              values={props.values}
+              switchToYaml={switchToYaml}
+            />
+          );
+        }}
       </Formik>
     ),
     [editorView, switchToUI, switchToYaml]
diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
index 6e35db96000bb..cd9665952c93f 100644
--- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
+++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx
@@ -1,5 +1,4 @@
 import { dump } from "js-yaml";
-import merge from "lodash/merge";
 import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
 import { useIntl } from "react-intl";
 import { useLocalStorage } from "react-use";
@@ -61,6 +60,7 @@ export const ConnectorBuilderFormStateProvider: React.FC<React.PropsWithChildren
   );
 
   const lastValidBuilderFormValuesRef = useRef<BuilderFormValues>(storedBuilderFormValues as BuilderFormValues);
+  const currentBuilderFormValuesRef = useRef<BuilderFormValues>(storedBuilderFormValues as BuilderFormValues);
 
   const setBuilderFormValues = useCallback(
     (values: BuilderFormValues, isValid: boolean) => {
@@ -68,14 +68,15 @@ export const ConnectorBuilderFormStateProvider: React.FC<React.PropsWithChildren
         // update ref first because calling setStoredBuilderFormValues might synchronously kick off a react render cycle.
         lastValidBuilderFormValuesRef.current = values;
       }
+      currentBuilderFormValuesRef.current = values;
       setStoredBuilderFormValues(values);
     },
     [setStoredBuilderFormValues]
   );
 
-  const builderFormValues = useMemo(() => {
-    return merge({}, DEFAULT_BUILDER_FORM_VALUES, storedBuilderFormValues);
-  }, [storedBuilderFormValues]);
+  // use the ref for the current builder form values because useLocalStorage will always serialize and deserialize the whole object,
+  // changing all the references which re-triggers all memoizations
+  const builderFormValues = currentBuilderFormValuesRef.current || DEFAULT_BUILDER_FORM_VALUES;
 
   const [jsonManifest, setJsonManifest] = useLocalStorage<ConnectorManifest>(
     "connectorBuilderJsonManifest",
@@ -119,9 +120,9 @@ export const ConnectorBuilderFormStateProvider: React.FC<React.PropsWithChildren
       editorView !== "ui"
         ? jsonManifest
         : builderFormValues === lastValidBuilderFormValues
-        ? jsonManifest
+        ? derivedJsonManifest
         : convertToManifest(lastValidBuilderFormValues),
-    [builderFormValues, editorView, jsonManifest, lastValidBuilderFormValues]
+    [builderFormValues, editorView, jsonManifest, derivedJsonManifest, lastValidBuilderFormValues]
   );
 
   const [selectedView, setSelectedView] = useState<BuilderView>("global");