From 610e668cc249ab38f813f230d1bae393153b0e9c Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Thu, 22 Dec 2022 03:07:45 -0800 Subject: [PATCH] [Connector Builder] Add stream slicer (#20748) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector * make connector form work in connector builder * wip * fix small stuff * add basic input UI * user inputs * make most of inputs configuration work * fix a bunch of stuff * handle unknown config types * add warning label * fix label * fix some styling * review comments * improve state management and error handling * allow auth configuration * check for conflicts with the inferred inputs * fix invisible inputs * handle stored form values that don't contain new fields properly * session token and oauth authentication * fill in session token variable * fix merge of default values * add primaryKey and cursorField to builder types, and consolidate default valeues to types.ts * add cursor and primary key fields to ui * save * add page size and token option inputs * fixes after rebase * add pagination * fix pagination types * handle empty field_name better * Update airbyte-webapp/src/locales/en.json Co-authored-by: Lake Mossman * Update airbyte-webapp/src/components/connectorBuilder/Builder/InputsView.tsx Co-authored-by: Lake Mossman * inputs editing weirdness * input form reset * using the Label component * reduce redundancy and hide advanced input options for inferred inputs * unnecessary validation * typo * unnecessary effect hook * build spec even for invalid forms but do not update stream list * typos * make sure validation error does not go away * make primary key and cursor optional, and reorder * save toggle group progress * fix style of toggle label * handle empty values better * fix page size/token option field validation and rendering * handle cursor pagination page size option correctly * save stream slicer progress * finish stream slicer * fix stream slicer fields and validation Co-authored-by: Joe Reuter --- .../connectorBuilder/Builder/BuilderField.tsx | 7 +- .../Builder/StreamConfigView.tsx | 2 + .../Builder/StreamSlicerSection.tsx | 182 ++++++++++++++++++ .../src/components/connectorBuilder/types.ts | 81 +++++++- 4 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/StreamSlicerSection.tsx diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx index 30171879af3f..4b2443affc7b 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -32,6 +32,7 @@ interface BaseFieldProps { tooltip?: string; readOnly?: boolean; optional?: boolean; + pattern?: RegExp; } type BuilderFieldProps = BaseFieldProps & @@ -66,6 +67,7 @@ export const BuilderField: React.FC = ({ tooltip, optional = false, readOnly, + pattern, ...props }) => { const [field, meta, helpers] = useField(path); @@ -116,7 +118,10 @@ export const BuilderField: React.FC = ({ )} {hasError && ( - + )} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx index 09225da07c86..1171aac969e4 100644 --- a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -16,6 +16,7 @@ import { BuilderTitle } from "./BuilderTitle"; import { KeyValueListField } from "./KeyValueListField"; import { PaginationSection } from "./PaginationSection"; import styles from "./StreamConfigView.module.scss"; +import { StreamSlicerSection } from "./StreamSlicerSection"; interface StreamConfigViewProps { streamNum: number; @@ -97,6 +98,7 @@ export const StreamConfigView: React.FC = ({ streamNum }) /> + string; +} + +export const StreamSlicerSection: React.FC = ({ streamFieldPath }) => { + const [field, , helpers] = useField(streamFieldPath("streamSlicer")); + + const handleToggle = (newToggleValue: boolean) => { + if (newToggleValue) { + helpers.setValue({ + type: "ListStreamSlicer", + slice_values: [], + cursor_field: "", + }); + } else { + helpers.setValue(undefined); + } + }; + const toggledOn = field.value !== undefined; + + return ( + + ), + toggledOn, + onToggle: handleToggle, + }} + > + + + + + + + + ), + }, + { + label: "Datetime", + typeValue: "DatetimeStreamSlicer", + children: ( + <> + + + + + + + + + + + + + + + + + + ), + }, + ]} + /> + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 57d0c1f0dd01..f56dc225e156 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -15,6 +15,7 @@ import { SessionTokenAuthenticator, DefaultPaginatorAllOfPaginationStrategy, RequestOption, + SimpleRetrieverAllOfStreamSlicer, } from "core/request/ConnectorManifest"; export interface BuilderFormInput { @@ -61,6 +62,7 @@ export interface BuilderStream { pageTokenOption: RequestOption; pageSizeOption?: RequestOption; }; + streamSlicer?: SimpleRetrieverAllOfStreamSlicer; } export const DEFAULT_BUILDER_FORM_VALUES: BuilderFormValues = { @@ -213,6 +215,17 @@ export function getInferredInputs(values: BuilderFormValues): BuilderFormInput[] } export const injectIntoValues = ["request_parameter", "header", "path", "body_data", "body_json"]; +const nonPathRequestOptionSchema = yup + .object() + .shape({ + inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), + field_name: yup.string().required("form.empty.error"), + }) + .notRequired() + .default(undefined); + +// eslint-disable-next-line no-useless-escape +export const timeDeltaRegex = /^(([\.\d]+?)y)?(([\.\d]+?)m)?(([\.\d]+?)w)?(([\.\d]+?)d)?$/; export const builderFormValidationSchema = yup.object().shape({ global: yup.object().shape({ @@ -261,14 +274,7 @@ export const builderFormValidationSchema = yup.object().shape({ paginator: yup .object() .shape({ - pageSizeOption: yup - .object() - .shape({ - inject_into: yup.mixed().oneOf(injectIntoValues.filter((val) => val !== "path")), - field_name: yup.string().required("form.empty.error"), - }) - .notRequired() - .default(undefined), + pageSizeOption: nonPathRequestOptionSchema, pageTokenOption: yup.object().shape({ inject_into: yup.mixed().oneOf(injectIntoValues), field_name: yup.mixed().when("inject_into", { @@ -305,6 +311,64 @@ export const builderFormValidationSchema = yup.object().shape({ }) .notRequired() .default(undefined), + streamSlicer: yup + .object() + .shape({ + cursor_field: yup.string().required("form.empty.error"), + slice_values: yup.mixed().when("type", { + is: "ListStreamSlicer", + then: yup.array().of(yup.string()), + otherwise: (schema) => schema.strip(), + }), + request_option: nonPathRequestOptionSchema, + start_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + end_datetime: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + step: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().matches(timeDeltaRegex, "form.pattern.error").required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + datetime_format: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string().required("form.empty.error"), + otherwise: (schema) => schema.strip(), + }), + start_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + end_time_option: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: nonPathRequestOptionSchema, + otherwise: (schema) => schema.strip(), + }), + stream_state_field_start: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + stream_state_field_end: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + lookback_window: yup.mixed().when("type", { + is: "DatetimeStreamSlicer", + then: yup.string(), + otherwise: (schema) => schema.strip(), + }), + }) + .notRequired() + .default(undefined), }) ), }); @@ -369,6 +433,7 @@ export const convertToManifest = (values: BuilderFormValues): PatchedConnectorMa url_base: values.global?.urlBase, } : { type: "NoPagination" }, + stream_slicer: stream.streamSlicer, config: {}, }, config: {},