[Connector Builder] Add paginator

airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx

@@ -1,4 +1,5 @@
 import { Form, Formik, useField } from "formik";
+import merge from "lodash/merge";
 import { useState } from "react";
 import React from "react";
 import { FormattedMessage, useIntl } from "react-intl";
@@ -9,7 +10,7 @@ import { Modal, ModalBody, ModalFooter } from "components/ui/Modal";
 import { FormikPatch } from "core/form/FormikPatch";
 
 import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg";
-import { BuilderStream } from "../types";
+import { BuilderStream, DEFAULT_BUILDER_STREAM_VALUES } from "../types";
 import styles from "./AddStreamButton.module.scss";
 import { BuilderField } from "./BuilderField";
 
@@ -49,18 +50,11 @@ export const AddStreamButton: React.FC<AddStreamButtonProps> = ({ onAddStream, b
           onSubmit={(values: AddStreamValues) => {
             helpers.setValue([
               ...streamsField.value,
-              {
-                fieldPointer: [],
-                httpMethod: "GET",
-                requestOptions: {
-                  requestParameters: [],
-                  requestHeaders: [],
-                  requestBody: [],
-                },
+              merge({}, DEFAULT_BUILDER_STREAM_VALUES, {
                 ...initialValues,
                 name: values.streamName,
                 urlPath: values.urlPath,
-              },
+              }),
             ]);
             setIsOpen(false);
             onAddStream(numStreams);

airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss

@@ -18,4 +18,5 @@
 .form {
   flex: 1;
   padding: variables.$spacing-xl;
+  overflow: auto;
 }

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss

@@ -6,3 +6,8 @@
   flex-direction: column;
   gap: variables.$spacing-xl;
 }
+
+.toggleContainer {
+  display: flex;
+  gap: variables.$spacing-md;
+}

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx

@@ -2,13 +2,38 @@ import classNames from "classnames";
 import React from "react";
 
 import { Card } from "components/ui/Card";
+import { CheckBox } from "components/ui/CheckBox";
 
 import styles from "./BuilderCard.module.scss";
 
 interface BuilderCardProps {
   className?: string;
+  toggleConfig?: {
+    label: React.ReactNode;
+    toggledOn: boolean;
+    onToggle: (newToggleValue: boolean) => void;
+  };
 }
 
-export const BuilderCard: React.FC<React.PropsWithChildren<BuilderCardProps>> = ({ children, className }) => {
-  return <Card className={classNames(className, styles.card)}>{children}</Card>;
+export const BuilderCard: React.FC<React.PropsWithChildren<BuilderCardProps>> = ({
+  children,
+  className,
+  toggleConfig,
+}) => {
+  return (
+    <Card className={classNames(className, styles.card)}>
+      {toggleConfig && (
+        <div className={styles.toggleContainer}>
+          <CheckBox
+            checked={toggleConfig.toggledOn}
+            onChange={(event) => {
+              toggleConfig.onToggle(event.target.checked);
+            }}
+          />
+          {toggleConfig.label}
+        </div>
+      )}
+      {(!toggleConfig || toggleConfig.toggledOn) && children}
+    </Card>
+  );
 };

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx

@@ -97,6 +97,9 @@ export const BuilderField: React.FC<BuilderFieldProps> = ({
           {...field}
           onChange={(e) => {
             field.onChange(e);
+            if (e.target.value === "") {
+              helpers.setValue(undefined);
+            }
             props.onChange?.(e.target.value);
           }}
           type={props.type}

airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx

@@ -10,14 +10,10 @@ import { Heading } from "components/ui/Heading";
 import { Text } from "components/ui/Text";
 
 import { useConfirmationModalService } from "hooks/services/ConfirmationModal";
-import {
-  BuilderView,
-  DEFAULT_BUILDER_FORM_VALUES,
-  useConnectorBuilderState,
-} from "services/connectorBuilder/ConnectorBuilderStateService";
+import { BuilderView, useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService";
 
 import { DownloadYamlButton } from "../DownloadYamlButton";
-import { BuilderFormValues, getInferredInputs } from "../types";
+import { BuilderFormValues, DEFAULT_BUILDER_FORM_VALUES, getInferredInputs } from "../types";
 import { useBuilderErrors } from "../useBuilderErrors";
 import { AddStreamButton } from "./AddStreamButton";
 import styles from "./BuilderSidebar.module.scss";

airbyte-webapp/src/components/connectorBuilder/Builder/InjectRequestOptionFields.tsx

@@ -0,0 +1,49 @@
+import { useField } from "formik";
+
+import { RequestOption } from "core/request/ConnectorManifest";
+
+import { injectIntoValues } from "../types";
+import { BuilderField } from "./BuilderField";
+
+interface InjectRequestOptionFieldsProps {
+  path: string;
+  descriptor: string;
+  excludeInjectIntoValues?: string[];
+}
+
+export const InjectRequestOptionFields: React.FC<InjectRequestOptionFieldsProps> = ({
+  path,
+  descriptor,
+  excludeInjectIntoValues,
+}) => {
+  const [field, , helpers] = useField<RequestOption>(path);
+
+  return (
+    <>
+      <BuilderField
+        type="enum"
+        path={`${path}.inject_into`}
+        options={
+          excludeInjectIntoValues
+            ? injectIntoValues.filter((val) => !excludeInjectIntoValues.includes(val))
+            : injectIntoValues
+        }
+        onChange={(newValue) => {
+          if (newValue === "path") {
+            helpers.setValue({ inject_into: newValue, field_name: undefined });
+          }
+        }}
+        label="Inject into"
+        tooltip={`Configures where the ${descriptor} should be set on the HTTP requests`}
+      />
+      {field.value.inject_into !== "path" && (
+        <BuilderField
+          type="string"
+          path={`${path}.field_name`}
+          label="Field name"
+          tooltip={`Configures which key should be used in the location that the ${descriptor} is being injected into`}
+        />
+      )}
+    </>
+  );
+};

airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx

@@ -334,7 +334,7 @@ const InputItem = ({
   isInferredInput: boolean;
 }): JSX.Element => {
   return (
-    <li className={styles.listItem}>
+    <li className={styles.listItem} key={input.key}>
       <div className={styles.itemLabel}>{input.definition.title || input.key}</div>
       <Button
         className={styles.itemButton}

airbyte-webapp/src/components/connectorBuilder/Builder/PaginationSection.tsx

@@ -0,0 +1,164 @@
+import { useField } from "formik";
+
+import GroupControls from "components/GroupControls";
+import { ControlLabels } from "components/LabeledControl";
+
+import { BuilderCard } from "./BuilderCard";
+import { BuilderField } from "./BuilderField";
+import { BuilderOneOf } from "./BuilderOneOf";
+import { InjectRequestOptionFields } from "./InjectRequestOptionFields";
+import { ToggleGroupField } from "./ToggleGroupField";
+
+interface PaginationSectionProps {
+  streamFieldPath: (fieldPath: string) => string;
+}
+
+export const PaginationSection: React.FC<PaginationSectionProps> = ({ streamFieldPath }) => {
+  const [field, , helpers] = useField(streamFieldPath("paginator"));
+  const [pageSizeField] = useField(streamFieldPath("paginator.strategy.page_size"));
+  const [, , pageSizeOptionHelpers] = useField(streamFieldPath("paginator.pageSizeOption"));
+
+  const handleToggle = (newToggleValue: boolean) => {
+    if (newToggleValue) {
+      helpers.setValue({
+        strategy: {
+          type: "OffsetIncrement",
+        },
+        pageTokenOption: {
+          inject_into: "request_parameter",
+        },
+      });
+    } else {
+      helpers.setValue(undefined);
+    }
+  };
+  const toggledOn = field.value !== undefined;
+
+  const pageTokenOption = (
+    <GroupControls
+      label={
+        <ControlLabels
+          label="Page token option"
+          infoTooltipContent="Configures how the page token will be sent in requests to the source API"
+        />
+      }
+    >
+      <InjectRequestOptionFields path={streamFieldPath("paginator.pageTokenOption")} descriptor="page token" />
+    </GroupControls>
+  );
+
+  const pageSizeOption = (
+    <ToggleGroupField
+      label="Page size option"
+      tooltip="Configures how the page size will be sent in requests to the source API"
+      fieldPath={streamFieldPath("paginator.pageSizeOption")}
+      initialValues={{
+        inject_into: "request_parameter",
+        field_name: "",
+      }}
+    >
+      <InjectRequestOptionFields
+        path={streamFieldPath("paginator.pageSizeOption")}
+        descriptor="page size"
+        excludeInjectIntoValues={["path"]}
+      />
+    </ToggleGroupField>
+  );
+
+  return (
+    <BuilderCard
+      toggleConfig={{
+        label: (
+          <ControlLabels
+            label="Pagination"
+            infoTooltipContent="Configure how pagination is handled by your connector"
+          />
+        ),
+        toggledOn,
+        onToggle: handleToggle,
+      }}
+    >
+      <BuilderOneOf
+        path={streamFieldPath("paginator.strategy")}
+        label="Mode"
+        tooltip="Pagination method to use for requests sent to the API"
+        options={[
+          {
+            label: "Offset Increment",
+            typeValue: "OffsetIncrement",
+            children: (
+              <>
+                <BuilderField
+                  type="number"
+                  path={streamFieldPath("paginator.strategy.page_size")}
+                  label="Page size"
+                  tooltip="Set the size of each page"
+                />
+                {pageSizeOption}
+                {pageTokenOption}
+              </>
+            ),
+          },
+          {
+            label: "Page Increment",
+            typeValue: "PageIncrement",
+            children: (
+              <>
+                <BuilderField
+                  type="number"
+                  path={streamFieldPath("paginator.strategy.page_size")}
+                  label="Page size"
+                  tooltip="Set the size of each page"
+                />
+                <BuilderField
+                  type="number"
+                  path={streamFieldPath("paginator.strategy.start_from_page")}
+                  label="Start from page"
+                  tooltip="Page number to start requesting pages from"
+                  optional
+                />
+                {pageSizeOption}
+                {pageTokenOption}
+              </>
+            ),
+          },
+          {
+            label: "Cursor Pagination",
+            typeValue: "CursorPagination",
+            children: (
+              <>
+                <BuilderField
+                  type="string"
+                  path={streamFieldPath("paginator.strategy.cursor_value")}
+                  label="Cursor value"
+                  tooltip="Value of the cursor to send in requests to the API"
+                />
+                <BuilderField
+                  type="string"
+                  path={streamFieldPath("paginator.strategy.stop_condition")}
+                  label="Stop condition"
+                  tooltip="Condition that determines when to stop requesting further pages"
+                  optional
+                />
+                <BuilderField
+                  type="number"
+                  path={streamFieldPath("paginator.strategy.page_size")}
+                  onChange={(newValue) => {
+                    if (newValue === undefined || newValue === "") {
+                      pageSizeOptionHelpers.setValue(undefined);
+                    }
+                  }}
+                  label="Page size"
+                  tooltip="Set the size of each page"
+                  optional
+                />
+                {pageSizeField.value && pageSizeField.value !== "" && pageSizeOption}
+                {pageTokenOption}
+              </>
+            ),
+          },
+        ]}
+      />
+    </BuilderCard>
+  );
+};

airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx

@@ -14,6 +14,7 @@ import { BuilderConfigView } from "./BuilderConfigView";
 import { BuilderField } from "./BuilderField";
 import { BuilderTitle } from "./BuilderTitle";
 import { KeyValueListField } from "./KeyValueListField";
+import { PaginationSection } from "./PaginationSection";
 import styles from "./StreamConfigView.module.scss";
 
 interface StreamConfigViewProps {
@@ -84,10 +85,18 @@ export const StreamConfigView: React.FC<StreamConfigViewProps> = ({ streamNum })
         <BuilderField
           type="array"
           path={streamFieldPath("fieldPointer")}
-          label="Field Pointer"
+          label="Record selector"
           tooltip="Pointer into the response that should be extracted as the final record"
         />
+        <BuilderField
+          type="array"
+          path={streamFieldPath("primaryKey")}
+          label="Primary key"
+          tooltip="Pointer into the response that should be used as the primary key when deduplicating records in the destination"
+          optional
+        />
       </BuilderCard>
+      <PaginationSection streamFieldPath={streamFieldPath} />
       <BuilderCard>
         <KeyValueListField
           path={streamFieldPath("requestOptions.requestParameters")}

airbyte-webapp/src/components/connectorBuilder/Builder/ToggleGroupField.module.scss

@@ -0,0 +1,12 @@
+@use "scss/variables";
+
+.label {
+  display: flex;
+  align-items: center;
+  gap: variables.$spacing-md;
+  height: 34px;
+
+  label {
+    padding-bottom: 0;
+  }
+}