diff --git a/airbyte-webapp/src/components/GroupControls/index.stories.tsx b/airbyte-webapp/src/components/GroupControls/index.stories.tsx index 414aec31cac6..3d99975add51 100644 --- a/airbyte-webapp/src/components/GroupControls/index.stories.tsx +++ b/airbyte-webapp/src/components/GroupControls/index.stories.tsx @@ -38,16 +38,18 @@ const propTwoFormBlock: FormBlock = { }; const conditionFormField: FormConditionItem = { - conditions: { - ChoiceOne: { + conditions: [ + { isRequired: true, _type: "formGroup", fieldKey: "choice_one_key", path: "section.conditional.choice_one", - jsonSchema: {}, properties: [propOneFormBlock, propTwoFormBlock], }, - }, + ], + selectionPath: "section.conditional.choice_one.type", + selectionKey: "type", + selectionConstValues: ["one"], isRequired: true, _type: "formCondition", fieldKey: "field_key", diff --git a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx index 3ce5d2256169..89eb0947b919 100644 --- a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx +++ b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx @@ -5,7 +5,9 @@ import { NavigateFunction, useNavigate } from "react-router-dom"; import { useLocation } from "react-use"; import { LocationSensorState } from "react-use/lib/useLocation"; +import { isFormBuildError } from "core/form/FormBuildError"; import { isVersionError } from "core/request/VersionError"; +import { TrackErrorFn, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { ErrorOccurredView } from "views/common/ErrorOccurredView"; import { ResourceNotFoundErrorBoundary } from "views/common/ResorceNotFoundErrorBoundary"; import { StartOverErrorView } from "views/common/StartOverErrorView"; @@ -21,6 +23,7 @@ interface ApiErrorBoundaryState { enum ErrorId { VersionMismatch = "version.mismatch", + FormBuild = "form.build", ServerUnavailable = "server.unavailable", UnknownError = "unknown", } @@ -29,6 +32,7 @@ interface ApiErrorBoundaryHookProps { location: LocationSensorState; onRetry?: () => void; navigate: NavigateFunction; + trackError: TrackErrorFn; } interface ApiErrorBoundaryProps { @@ -51,6 +55,10 @@ class ApiErrorBoundaryComponent extends React.Component< return { errorId: ErrorId.VersionMismatch, message: error.message }; } + if (isFormBuildError(error)) { + return { errorId: ErrorId.FormBuild, message: error.message }; + } + const isNetworkBoundaryMessage = error.message === "Failed to fetch"; const is502 = error.status === 502; @@ -72,6 +80,15 @@ class ApiErrorBoundaryComponent extends React.Component< } } + componentDidCatch(error: { message: string; status?: number; __type?: string }) { + if (isFormBuildError(error)) { + this.props.trackError(error, { + id: "formBuildError", + connectorDefinitionId: error.connectorDefinitionId, + }); + } + } + retry = () => { this.setState((state) => ({ didRetry: true, @@ -89,6 +106,21 @@ class ApiErrorBoundaryComponent extends React.Component< return ; } + if (errorId === ErrorId.FormBuild) { + return ( + + +
+ + + } + docLink="https://docs.airbyte.com/connector-development/connector-specification-reference/#airbyte-modifications-to-jsonschema" + /> + ); + } + if (errorId === ErrorId.ServerUnavailable && !didRetry) { return ; } @@ -111,9 +143,16 @@ export const ApiErrorBoundary: React.FC + {children} ); diff --git a/airbyte-webapp/src/core/domain/catalog/traverseSchemaToField.test.ts b/airbyte-webapp/src/core/domain/catalog/traverseSchemaToField.test.ts index 1411a2a4aa5e..f976c03e109a 100644 --- a/airbyte-webapp/src/core/domain/catalog/traverseSchemaToField.test.ts +++ b/airbyte-webapp/src/core/domain/catalog/traverseSchemaToField.test.ts @@ -1,4 +1,4 @@ -import { AirbyteJSONSchema } from "core/jsonSchema"; +import { AirbyteJSONSchema } from "core/jsonSchema/types"; import { traverseSchemaToField } from "./traverseSchemaToField"; diff --git a/airbyte-webapp/src/core/domain/connection/types.ts b/airbyte-webapp/src/core/domain/connection/types.ts index ba962af18100..c15bad7b1f4b 100644 --- a/airbyte-webapp/src/core/domain/connection/types.ts +++ b/airbyte-webapp/src/core/domain/connection/types.ts @@ -1,4 +1,4 @@ -import { AirbyteJSONSchema } from "core/jsonSchema"; +import { AirbyteJSONSchema } from "core/jsonSchema/types"; type ConnectionConfiguration = unknown; diff --git a/airbyte-webapp/src/core/form/FormBuildError.ts b/airbyte-webapp/src/core/form/FormBuildError.ts new file mode 100644 index 000000000000..a0de3a8fd381 --- /dev/null +++ b/airbyte-webapp/src/core/form/FormBuildError.ts @@ -0,0 +1,11 @@ +export class FormBuildError extends Error { + __type = "form.build"; + + constructor(public message: string, public connectorDefinitionId?: string) { + super(message); + } +} + +export function isFormBuildError(error: { __type?: string }): error is FormBuildError { + return error.__type === "form.build"; +} diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts b/airbyte-webapp/src/core/form/schemaToFormBlock.test.ts similarity index 71% rename from airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts rename to airbyte-webapp/src/core/form/schemaToFormBlock.test.ts index a53dc531cb32..12cdd43e0cfe 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts +++ b/airbyte-webapp/src/core/form/schemaToFormBlock.test.ts @@ -1,7 +1,7 @@ import { FormGroupItem } from "core/form/types"; +import { AirbyteJSONSchemaDefinition } from "core/jsonSchema/types"; -import { jsonSchemaToUiWidget } from "./schemaToUiWidget"; -import { AirbyteJSONSchemaDefinition } from "./types"; +import { jsonSchemaToFormBlock } from "./schemaToFormBlock"; it("should reformat jsonSchema to internal widget representation", () => { const schema: AirbyteJSONSchemaDefinition = { @@ -26,40 +26,13 @@ it("should reformat jsonSchema to internal widget representation", () => { }, }; - const builtSchema = jsonSchemaToUiWidget(schema, "key"); + const builtSchema = jsonSchemaToFormBlock(schema, "key"); const expected = { _type: "formGroup", path: "key", fieldKey: "key", isRequired: false, - jsonSchema: { - properties: { - dbname: { - description: "Name of the database.", - type: "string", - }, - host: { - description: "Hostname of the database.", - type: "string", - }, - password: { - airbyte_secret: true, - description: "Password associated with the username.", - type: "string", - }, - port: { - description: "Port of the database.", - type: "integer", - }, - user: { - description: "Username to use to access the database.", - type: "string", - }, - }, - required: ["host", "port", "user", "dbname"], - type: "object", - }, properties: [ { _type: "formItem", @@ -128,7 +101,7 @@ it("should turn single enum into const but keep multi value enum", () => { }, }; - const builtSchema = jsonSchemaToUiWidget(schema, "key"); + const builtSchema = jsonSchemaToFormBlock(schema, "key"); const expectedProperties = [ { @@ -176,7 +149,7 @@ it("should reformat jsonSchema to internal widget representation with parent sch }, }; - const builtSchema = jsonSchemaToUiWidget(schema, "key", undefined, { + const builtSchema = jsonSchemaToFormBlock(schema, "key", undefined, { required: ["key"], }); @@ -185,17 +158,6 @@ it("should reformat jsonSchema to internal widget representation with parent sch fieldKey: "key", path: "key", isRequired: true, - jsonSchema: { - properties: { - host: { - description: "Hostname of the database.", - type: "string", - }, - }, - required: ["host", "port", "user", "dbname"], - title: "Postgres Source Spec", - type: "object", - }, properties: [ { _type: "formItem", @@ -235,6 +197,11 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" api_key: { type: "string", }, + type: { + type: "string", + const: "api", + default: "api", + }, }, }, { @@ -245,6 +212,11 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" type: "string", examples: ["https://api.hubspot.com/"], }, + type: { + type: "string", + const: "oauth", + default: "oauth", + }, }, }, ], @@ -252,42 +224,12 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" }, }; - const builtSchema = jsonSchemaToUiWidget(schema, "key", undefined, { + const builtSchema = jsonSchemaToFormBlock(schema, "key", undefined, { required: ["key"], }); const expected = { _type: "formGroup", - jsonSchema: { - type: "object", - required: ["start_date", "credentials"], - properties: { - start_date: { type: "string" }, - credentials: { - type: "object", - description: "Credentials Condition Description", - title: "Credentials Condition", - order: 0, - oneOf: [ - { - title: "api key", - required: ["api_key"], - properties: { api_key: { type: "string" } }, - }, - { - title: "oauth", - required: ["redirect_uri"], - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, - ], - }, - }, - }, path: "key", fieldKey: "key", properties: [ @@ -307,16 +249,13 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" title: "Credentials Condition", order: 0, fieldKey: "credentials", - conditions: { - "api key": { + selectionConstValues: ["api", "oauth"], + selectionKey: "type", + selectionPath: "key.credentials.type", + conditions: [ + { title: "api key", _type: "formGroup", - jsonSchema: { - title: "api key", - required: ["api_key"], - type: "object", - properties: { api_key: { type: "string" } }, - }, path: "key.credentials", fieldKey: "credentials", properties: [ @@ -329,23 +268,24 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" multiline: false, type: "string", }, + { + _type: "formItem", + const: "api", + default: "api", + fieldKey: "type", + format: undefined, + isRequired: false, + isSecret: false, + multiline: false, + path: "key.credentials.type", + type: "string", + }, ], isRequired: false, }, - oauth: { + { title: "oauth", _type: "formGroup", - jsonSchema: { - title: "oauth", - required: ["redirect_uri"], - type: "object", - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, path: "key.credentials", fieldKey: "credentials", properties: [ @@ -359,10 +299,22 @@ it("should reformat jsonSchema to internal widget representation when has oneOf" multiline: false, type: "string", }, + { + _type: "formItem", + const: "oauth", + default: "oauth", + fieldKey: "type", + format: undefined, + isRequired: false, + isSecret: false, + multiline: false, + path: "key.credentials.type", + type: "string", + }, ], isRequired: false, }, - }, + ], isRequired: true, }, ], diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts b/airbyte-webapp/src/core/form/schemaToFormBlock.ts similarity index 59% rename from airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts rename to airbyte-webapp/src/core/form/schemaToFormBlock.ts index 360105af13b0..8f587c4b847b 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts +++ b/airbyte-webapp/src/core/form/schemaToFormBlock.ts @@ -1,9 +1,12 @@ +import { JSONSchema7Type } from "json-schema"; +import intersection from "lodash/intersection"; import pick from "lodash/pick"; import { FormBlock } from "core/form/types"; +import { AirbyteJSONSchemaDefinition, AirbyteJSONSchema } from "core/jsonSchema/types"; import { isDefined } from "utils/common"; -import { AirbyteJSONSchemaDefinition, AirbyteJSONSchema } from "./types"; +import { FormBuildError } from "./FormBuildError"; /** * Returns {@link FormBlock} representation of jsonSchema @@ -16,7 +19,7 @@ import { AirbyteJSONSchemaDefinition, AirbyteJSONSchema } from "./types"; * @param path * @param parentSchema */ -export const jsonSchemaToUiWidget = ( +export const jsonSchemaToFormBlock = ( jsonSchema: AirbyteJSONSchemaDefinition, key = "", path: string = key, @@ -37,20 +40,49 @@ export const jsonSchemaToUiWidget = ( } if (jsonSchema.oneOf?.length && jsonSchema.oneOf.length > 0) { - const conditions = Object.fromEntries( - jsonSchema.oneOf.map((condition) => { - if (typeof condition === "boolean") { - return []; - } - return [condition.title, jsonSchemaToUiWidget({ ...condition, type: jsonSchema.type }, key, path)]; - }) - ); + let possibleConditionSelectionKeys = null as string[] | null; + const conditions = jsonSchema.oneOf.map((condition) => { + if (typeof condition === "boolean") { + throw new FormBuildError("connectorForm.error.oneOfWithNonObjects"); + } + const formBlock = jsonSchemaToFormBlock({ ...condition, type: jsonSchema.type }, key, path); + if (formBlock._type !== "formGroup") { + throw new FormBuildError("connectorForm.error.oneOfWithNonObjects"); + } + + const constProperties = formBlock.properties + .filter((property) => property.const) + .map((property) => property.fieldKey); + + if (!possibleConditionSelectionKeys) { + // if this is the first condition, all const properties are candidates + possibleConditionSelectionKeys = constProperties; + } else { + // if there are candidates already, intersect with the const properties of the current condition + possibleConditionSelectionKeys = intersection(possibleConditionSelectionKeys, constProperties); + } + return formBlock; + }); + + if (!possibleConditionSelectionKeys?.length) { + // no shared const property in oneOf. This should never happen per specification, fail hard + throw new FormBuildError("connectorForm.error.oneOfWithoutConst"); + } + const selectionKey = possibleConditionSelectionKeys[0]; + const selectionPath = `${path}.${selectionKey}`; + // can't contain undefined values as we would have thrown on this with connectorForm.error.oneOfWithoutConst + const selectionConstValues = conditions.map( + (condition) => condition.properties.find((property) => property.path === selectionPath)?.const + ) as JSONSchema7Type[]; return { ...pickDefaultFields(jsonSchema), _type: "formCondition", path: path || key, fieldKey: key, + selectionPath, + selectionKey, + selectionConstValues, conditions, isRequired, }; @@ -67,20 +99,19 @@ export const jsonSchemaToUiWidget = ( _type: "objectArray", path: path || key, fieldKey: key, - properties: jsonSchemaToUiWidget(jsonSchema.items, key, path), + properties: jsonSchemaToFormBlock(jsonSchema.items, key, path), isRequired, }; } if (jsonSchema.type === "object") { const properties = Object.entries(jsonSchema.properties || []).map(([k, schema]) => - jsonSchemaToUiWidget(schema, k, path ? `${path}.${k}` : k, jsonSchema) + jsonSchemaToFormBlock(schema, k, path ? `${path}.${k}` : k, jsonSchema) ); return { ...pickDefaultFields(jsonSchema), _type: "formGroup", - jsonSchema, path: path || key, fieldKey: key, properties, diff --git a/airbyte-webapp/src/core/form/schemaToYup.test.ts b/airbyte-webapp/src/core/form/schemaToYup.test.ts new file mode 100644 index 000000000000..7d2b2f10ddbd --- /dev/null +++ b/airbyte-webapp/src/core/form/schemaToYup.test.ts @@ -0,0 +1,226 @@ +import * as yup from "yup"; + +import { AirbyteJSONSchema } from "core/jsonSchema/types"; + +import { jsonSchemaToFormBlock } from "./schemaToFormBlock"; +import { buildYupFormForJsonSchema } from "./schemaToYup"; + +// Note: We have to check yup schema with JSON.stringify +// as exactly same objects throw now equality due to `Received: serializes to the same string` error +it("should build schema for simple case", () => { + const schema: AirbyteJSONSchema = { + type: "object", + title: "Postgres Source Spec", + required: ["host", "port", "user", "dbname", "is_field_no_default"], + properties: { + host: { type: "string", description: "Hostname of the database." }, + port: { + type: "integer", + maximum: 65536, + minimum: 0, + description: "Port of the database.", + }, + user: { + type: "string", + description: "Username to use to access the database.", + }, + is_sandbox: { + type: "boolean", + default: false, + }, + is_field_no_default: { + type: "boolean", + }, + dbname: { type: "string", description: "Name of the database." }, + password: { + type: "string", + description: "Password associated with the username.", + }, + reports: { + type: "array", + items: { + type: "string", + }, + }, + }, + additionalProperties: false, + }; + const yupSchema = buildYupFormForJsonSchema(schema, jsonSchemaToFormBlock(schema)); + + const expectedSchema = yup.object().shape({ + host: yup.string().trim().required("form.empty.error").transform(String), + port: yup + .number() + .min(0) + .max(65536) + .required("form.empty.error") + .transform((val) => (isNaN(val) ? undefined : val)), + user: yup.string().trim().required("form.empty.error").transform(String), + is_sandbox: yup.boolean().default(false), + is_field_no_default: yup.boolean().required("form.empty.error"), + dbname: yup.string().trim().required("form.empty.error").transform(String), + password: yup.string().trim().transform(String), + reports: yup.array().of(yup.string().trim().transform(String)), + }); + + expect(JSON.stringify(yupSchema)).toEqual(JSON.stringify(expectedSchema)); +}); + +const simpleConditionalSchema: AirbyteJSONSchema = { + type: "object", + required: ["start_date", "credentials"], + properties: { + start_date: { + type: "string", + }, + max_objects: { + type: "number", + }, + credentials: { + type: "object", + oneOf: [ + { + title: "api key", + required: ["type", "api_key"], + properties: { + api_key: { + type: "string", + pattern: "\\w{5}", + }, + type: { + type: "string", + const: "api", + default: "api", + }, + }, + }, + { + title: "oauth", + required: ["type", "redirect_uri"], + properties: { + redirect_uri: { + type: "string", + examples: ["https://api.hubspot.com/"], + }, + type: { + type: "string", + const: "oauth", + default: "oauth", + }, + }, + }, + ], + }, + }, +}; + +it("should build correct mixed schema structure for conditional case", () => { + const yupSchema = buildYupFormForJsonSchema(simpleConditionalSchema, jsonSchemaToFormBlock(simpleConditionalSchema)); + + const expectedSchema = yup.object().shape({ + start_date: yup.string().trim().required("form.empty.error").transform(String), + max_objects: yup.number().transform((x) => x), + credentials: yup.object().shape({ + type: yup.mixed(), + api_key: yup + .mixed() + // Using dummy callbacks for then and otherwise as this test only checks whether the yup schema is structured as expected, it's not asserting that it validates form values as expected. + .when("type", { is: "", then: (x) => x, otherwise: (x) => x }) + .when("type", { is: "", then: (x) => x, otherwise: (x) => x }), + redirect_uri: yup + .mixed() + .when("type", { is: "", then: (x) => x, otherwise: (x) => x }) + .when("type", { is: "", then: (x) => x, otherwise: (x) => x }), + }), + }); + + expect(JSON.parse(JSON.stringify(yupSchema))).toEqual(JSON.parse(JSON.stringify(expectedSchema))); +}); + +// These tests check whether the built yup schema validates as expected, it is not checking the structure +describe("yup schema validations", () => { + const yupSchema = buildYupFormForJsonSchema(simpleConditionalSchema, jsonSchemaToFormBlock(simpleConditionalSchema)); + it("enforces required props for selected condition", () => { + expect(() => { + yupSchema.validateSync({ + start_date: "2022", + max_objects: 5, + credentials: { + // api needs api_key, so this should fail + type: "api", + redirect_uri: "test", + }, + }); + }).toThrowError("form.empty.error"); + }); + + it("does not enforce additional contraints if the condition is selected", () => { + expect(() => { + yupSchema.validateSync({ + start_date: "2022", + max_objects: 5, + credentials: { + type: "oauth", + redirect_uri: "test", + // does not match the pattern, but it should not be validated + api_key: "X", + }, + }); + }).not.toThrowError(); + }); + + it("enforces additional contraints only if the condition is selected", () => { + expect(() => { + yupSchema.validateSync({ + start_date: "2022", + max_objects: 5, + credentials: { + type: "api", + // does not match the pattern, so it should fail + api_key: "X", + }, + }); + }).toThrowError("form.pattern.error"); + }); + + it("strips out properties belonging to other conditions", () => { + const cleanedValues = yupSchema.cast( + { + start_date: "2022", + max_objects: 5, + credentials: { + type: "api", + api_key: "X", + redirect_uri: "test", + }, + }, + { + stripUnknown: true, + } + ); + expect(cleanedValues.credentials).toEqual({ + type: "api", + api_key: "X", + }); + }); + + it("does not strip out any properties if the condition key is not set to prevent data loss of legacy specs", () => { + const cleanedValues = yupSchema.cast( + { + start_date: "2022", + max_objects: 5, + credentials: { + api_key: "X", + redirect_uri: "test", + }, + }, + { + stripUnknown: true, + } + ); + expect(cleanedValues.credentials).toEqual({ + api_key: "X", + redirect_uri: "test", + }); + }); +}); diff --git a/airbyte-webapp/src/core/form/schemaToYup.ts b/airbyte-webapp/src/core/form/schemaToYup.ts new file mode 100644 index 000000000000..eb19297e7d77 --- /dev/null +++ b/airbyte-webapp/src/core/form/schemaToYup.ts @@ -0,0 +1,226 @@ +import { JSONSchema7, JSONSchema7Type } from "json-schema"; +import * as yup from "yup"; + +import { FormBlock, FormGroupItem, FormObjectArrayItem, FormConditionItem } from "core/form/types"; +import { isDefined } from "utils/common"; + +import { FormBuildError } from "./FormBuildError"; + +/** + * Returns yup.schema for validation + * + * This method builds yup schema based on jsonSchema ${@link JSONSchema7} and the derived ${@link FormBlock}. + * Every property is walked through recursively in case it is condition | object | array. + * + * @param jsonSchema + * @param formField The corresponding {@link FormBlock} to the given schema + * @param parentSchema used in recursive schema building as required fields can be described in parentSchema + * @param propertyKey used in recursive schema building for building path for uiConfig + * @param propertyPath constructs path of property + */ +export const buildYupFormForJsonSchema = ( + jsonSchema: JSONSchema7, + formField: FormBlock, + parentSchema?: JSONSchema7, + propertyKey?: string, + propertyPath: string | undefined = propertyKey +): yup.AnySchema => { + let schema: + | yup.NumberSchema + | yup.StringSchema + | yup.AnyObjectSchema + | yup.ArraySchema + | yup.BooleanSchema + | null = null; + + if (jsonSchema.oneOf && propertyPath) { + const conditionFormField = formField as FormConditionItem; + // for all keys in all of the possible objects from the oneOf, collect the sub yup-schema in a map. + // the keys of the map are the keys of the property, the value is an array of the selection const value + // for the condition plus the sub schema for that property in that condition. + // As not all keys will show up in every condition, there can be a different number of possible sub schemas + // per key; at least one and at max the number of conditions (if a key is part of every oneOf) + // For example: + // If there are three possible schemas with the following properties: + // A: { type: "A", prop1: number, prop2: string } + // B: { type: "B", prop1: string, prop2: string } + // C: { type: "C", prop2: boolean } + // Then the map will look like this: + // { + // prop1: [["A", number], ["B", string]] + // prop2: [["A", string], ["B", string], ["C", boolean]] + // } + const flattenedKeys: Map> = new Map(); + jsonSchema.oneOf.forEach((condition, index) => { + if (typeof condition !== "object") { + throw new FormBuildError("connectorForm.error.oneOfWithNonObjects"); + } + const selectionConstValue = conditionFormField.selectionConstValues[index]; + const selectionFormField = conditionFormField.conditions[index]; + Object.entries(condition.properties || {}).forEach(([key, prop], propertyIndex) => { + if (!flattenedKeys.has(key)) { + flattenedKeys.set(key, []); + } + flattenedKeys + .get(key) + ?.push([ + selectionConstValue, + typeof prop === "boolean" + ? yup.bool() + : buildYupFormForJsonSchema( + prop, + selectionFormField.properties[propertyIndex], + condition, + key, + propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey + ), + ]); + }); + }); + const selectionKey = conditionFormField.selectionKey; + + // build the final object with all the keys - add "when" clauses to apply the + // right sub-schema depending on which selection const value is defined. + // if a key doesn't have a sub schema for a selection const value, set it to "strip" + // so it's removed before the form is sent to the server + // For example (the map from above): + // { + // prop1: number.when(type == "A"), string.when(type == "B"), strip.when(type neither "A" nor "B") + // prop2: string.when(type == "A"), string.when(type == "B"), boolean.when(type == "C"), strip.when(type neither "A" nor "B" nor "C") + // } + return yup.object().shape( + Object.fromEntries( + Array.from(flattenedKeys.entries()).map(([key, schemaByCondition]) => { + let mergedSchema = yup.mixed(); + if (key === selectionKey) { + // do not validate the selectionKey itself, as the user can't change it it doesnt matter + return [key, mergedSchema]; + } + const allSelectionConstValuesWithThisKey = schemaByCondition.map(([constValue]) => constValue); + schemaByCondition.forEach(([selectionConstValue, conditionalSchema]) => { + mergedSchema = mergedSchema.when(selectionKey, { + is: selectionConstValue, + then: () => conditionalSchema, + otherwise: (schema) => schema, + }); + }); + mergedSchema = mergedSchema.when(selectionKey, { + is: (val: JSONSchema7Type | undefined) => + // if typeof val is actually undefined, we are dealing with an inconsistent configuration which doesn't have any value for the condition key. + // in this case, just keep the existing value to prevent data loss. + typeof val !== "undefined" && !allSelectionConstValuesWithThisKey.includes(val), + then: (schema) => schema.strip(), + otherwise: (schema) => schema, + }); + return [key, mergedSchema]; + }) + ) + ); + } + + switch (jsonSchema.type) { + case "string": + schema = yup + .string() + .transform((val) => String(val)) + .trim(); + + if (jsonSchema?.pattern !== undefined) { + schema = schema.matches(new RegExp(jsonSchema.pattern), "form.pattern.error"); + } + + break; + case "boolean": + schema = yup.boolean(); + break; + case "integer": + case "number": + schema = yup.number().transform((value) => (isNaN(value) ? undefined : value)); + + if (jsonSchema?.minimum !== undefined) { + schema = schema.min(jsonSchema?.minimum); + } + + if (jsonSchema?.maximum !== undefined) { + schema = schema.max(jsonSchema?.maximum); + } + break; + case "array": + if (typeof jsonSchema.items === "object" && !Array.isArray(jsonSchema.items)) { + schema = yup + .array() + .of( + buildYupFormForJsonSchema( + jsonSchema.items, + (formField as FormObjectArrayItem).properties, + jsonSchema, + propertyKey, + propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey + ) + ); + } + break; + case "object": + let objectSchema = yup.object(); + + const keyEntries = Object.entries(jsonSchema.properties || {}).map(([propertyKey, propertySchema]) => { + const correspondingFormField = (formField as FormGroupItem).properties.find( + (property) => property.fieldKey === propertyKey + ); + if (!correspondingFormField) { + throw new Error("mismatch between form fields and schema"); + } + return [ + propertyKey, + typeof propertySchema !== "boolean" + ? buildYupFormForJsonSchema( + propertySchema, + correspondingFormField, + jsonSchema, + propertyKey, + propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey + ) + : yup.bool(), + ]; + }); + + if (keyEntries.length) { + objectSchema = objectSchema.shape(Object.fromEntries(keyEntries)); + } else { + objectSchema = objectSchema.default({}); + } + + schema = objectSchema; + } + + if (schema) { + const hasDefault = isDefined(jsonSchema.default); + + if (hasDefault) { + // @ts-expect-error can't infer correct type here so lets just use default from json_schema + schema = schema.default(jsonSchema.default); + } + + if (!hasDefault && jsonSchema.const) { + // @ts-expect-error can't infer correct type here so lets just use default from json_schema + schema = schema.oneOf([jsonSchema.const]).default(jsonSchema.const); + } + + if (jsonSchema.enum) { + // @ts-expect-error as enum is array we are going to use it as oneOf for yup + schema = schema.oneOf(jsonSchema.enum); + } + + const isRequired = + !hasDefault && + parentSchema && + Array.isArray(parentSchema?.required) && + parentSchema.required.find((item) => item === propertyKey); + + if (schema && isRequired) { + schema = schema.required("form.empty.error"); + } + } + + return schema || yup.mixed(); +}; diff --git a/airbyte-webapp/src/core/form/types.ts b/airbyte-webapp/src/core/form/types.ts index 272d4c14ffcc..05cd7240f12e 100644 --- a/airbyte-webapp/src/core/form/types.ts +++ b/airbyte-webapp/src/core/form/types.ts @@ -1,6 +1,6 @@ -import { JSONSchema7TypeName } from "json-schema"; +import { JSONSchema7Type, JSONSchema7TypeName } from "json-schema"; -import { AirbyteJSONSchema } from "core/jsonSchema"; +import { AirbyteJSONSchema } from "core/jsonSchema/types"; /** * When turning the JSON schema into `FormBlock`s, @@ -35,13 +35,24 @@ export interface FormBaseItem extends FormItem { export interface FormGroupItem extends FormItem { _type: "formGroup"; - jsonSchema: AirbyteJSONSchema; properties: FormBlock[]; } export interface FormConditionItem extends FormItem { _type: "formCondition"; - conditions: Record; + conditions: FormGroupItem[]; + /** + * The full path to the const property describing which condition is selected (e.g. connectionConfiguration.a.deep.path.type) + */ + selectionPath: string; + /** + * The key of the const property describing which condition is selected (e.g. type) + */ + selectionKey: string; + /** + * The possible values of the selectionKey property ordered in the same way as the conditions + */ + selectionConstValues: JSONSchema7Type[]; } export interface FormObjectArrayItem extends FormItem { @@ -50,10 +61,3 @@ export interface FormObjectArrayItem extends FormItem { } export type FormBlock = FormGroupItem | FormBaseItem | FormConditionItem | FormObjectArrayItem; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type WidgetConfig = Record; -export type WidgetConfigMap = Record; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type FormComponentOverrideProps = Record; diff --git a/airbyte-webapp/src/core/form/uiWidget.test.ts b/airbyte-webapp/src/core/form/uiWidget.test.ts deleted file mode 100644 index 722f180182ae..000000000000 --- a/airbyte-webapp/src/core/form/uiWidget.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { jsonSchemaToUiWidget } from "core/jsonSchema/schemaToUiWidget"; - -import { FormBlock } from "./types"; -import { buildPathInitialState } from "./uiWidget"; - -const formItems: FormBlock[] = [ - { - _type: "formGroup", - fieldKey: "key", - path: "key", - isRequired: true, - jsonSchema: { - type: "object", - required: ["start_date", "credentials"], - properties: { - start_date: { type: "string" }, - credentials: { - type: "object", - oneOf: [ - { - title: "api key", - required: ["api_key"], - properties: { api_key: { type: "string" } }, - }, - { - title: "oauth", - required: ["redirect_uri"], - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, - ], - }, - }, - }, - properties: [ - { - _type: "formItem", - fieldKey: "start_date", - path: "key.start_date", - isRequired: true, - type: "string", - }, - { - _type: "formCondition", - fieldKey: "credentials", - path: "key.credentials", - isRequired: true, - conditions: { - "api key": { - title: "api key", - _type: "formGroup", - fieldKey: "credentials", - path: "key.credentials", - isRequired: false, - jsonSchema: { - title: "api key", - required: ["api_key"], - properties: { api_key: { type: "string" } }, - }, - properties: [ - { - _type: "formItem", - fieldKey: "api_key", - path: "key.credentials.api_key", - isRequired: true, - type: "string", - }, - ], - }, - oauth: { - title: "oauth", - _type: "formGroup", - fieldKey: "credentials", - path: "key.credentials", - isRequired: false, - jsonSchema: { - title: "oauth", - required: ["redirect_uri"], - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, - properties: [ - { - _type: "formItem", - examples: ["https://api.hubspot.com/"], - fieldKey: "redirect_uri", - path: "key.credentials.redirect_uri", - isRequired: true, - type: "string", - }, - ], - }, - }, - }, - ], - }, -]; - -it("should select first key by default", () => { - const uiWidgetState = buildPathInitialState(formItems, {}); - expect(uiWidgetState).toEqual({ - "key.credentials": { - selectedItem: "api key", - }, - "key.credentials.api_key": {}, - "key.start_date": {}, - }); -}); - -it("should select key selected in default values", () => { - const uiWidgetState = buildPathInitialState( - formItems, - { - key: { - credentials: { - redirect_uri: "value", - }, - }, - }, - {} - ); - expect(uiWidgetState).toEqual({ - "key.credentials": { - selectedItem: "oauth", - }, - "key.credentials.redirect_uri": {}, - "key.start_date": {}, - }); -}); - -it("should select correct key for enum", () => { - const fields = jsonSchemaToUiWidget( - JSON.parse( - `{"type":"object","title":"File Source Spec","$schema":"http://json-schema.org/draft-07/schema#","required":["dataset_name","format","url","provider"],"properties":{"url":{"type":"string","description":"URL path to access the file to be replicated"},"format":{"enum":["csv","json","html","excel","feather","parquet","orc","pickle"],"type":"string","default":"csv","description":"File Format of the file to be replicated (Warning: some format may be experimental, please refer to docs)."},"provider":{"type":"object","oneOf":[{"title":"HTTPS: Public Web","required":["storage"],"properties":{"storage":{"enum":["HTTPS"],"type":"string","default":"HTTPS"}}},{"title":"GCS: Google Cloud Storage","required":["storage","reader_impl"],"properties":{"storage":{"enum":["GCS"],"type":"string","default":"GCS"},"reader_impl":{"enum":["smart_open","gcsfs"],"type":"string","default":"gcsfs","description":"This connector provides multiple methods to retrieve data from GCS using either smart-open python libraries or GCSFS"},"service_account_json":{"type":"string","description":"In order to access private Buckets stored on Google Cloud, this connector would need a service account json credentials with the proper permissions as described here. Please generate the credentials.json file and copy/paste its content to this field (expecting JSON formats). If accessing publicly available data, this field is not necessary."}}},{"title":"S3: Amazon Web Services","required":["storage","reader_impl"],"properties":{"storage":{"enum":["S3"],"type":"string","default":"S3"},"reader_impl":{"enum":["smart_open","s3fs"],"type":"string","default":"s3fs","description":"This connector provides multiple methods to retrieve data from AWS S3 using either smart-open python libraries or S3FS"},"aws_access_key_id":{"type":"string","description":"In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary."},"aws_secret_access_key":{"type":"string","description":"In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.","airbyte_secret":true}}},{"title":"SSH: Secure Shell","required":["storage","user","host"],"properties":{"host":{"type":"string"},"user":{"type":"string"},"storage":{"enum":["SSH"],"type":"string","default":"SSH"},"password":{"type":"string","airbyte_secret":true}}},{"title":"SFTP: Secure File Transfer Protocol","required":["storage","user","host"],"properties":{"host":{"type":"string"},"user":{"type":"string"},"storage":{"enum":["SFTP"],"type":"string","default":"SFTP"},"password":{"type":"string","airbyte_secret":true}}},{"title":"WebHDFS: HDFS REST API (Untested)","required":["storage","host","port"],"properties":{"host":{"type":"string"},"port":{"type":"number"},"storage":{"enum":["WebHDFS"],"type":"string","default":"WebHDFS","description":"WARNING: smart_open library provides the ability to stream files over this protocol but we haven't been able to test this as part of Airbyte yet, please use with caution. We would love to hear feedbacks from you if you are able or fail to use this!"}}},{"title":"Local Filesystem (limited)","required":["storage"],"properties":{"storage":{"enum":["local"],"type":"string","default":"local","description":"WARNING: Note that local storage URL available for read must start with the local mount \\"/local/\\" at the moment until we implement more advanced docker mounting options..."}}}],"default":"Public Web","description":"Storage Provider or Location of the file(s) to be replicated."},"dataset_name":{"type":"string","description":"Name of the final table where to replicate this file (should include only letters, numbers dash and underscores)"},"reader_options":{"type":"string","examples":["{}","{'sep': ' '}"],"description":"This should be a valid JSON string used by each reader/parser to provide additional options and tune its behavior"}},"additionalProperties":false}` - ), - "key" - ); - - const uiWidgetState = buildPathInitialState( - [fields], - { - key: { - provider: { - storage: "GCS", - }, - }, - }, - {} - ); - expect(uiWidgetState).toEqual({ - "key.dataset_name": {}, - "key.format": { - default: "csv", - }, - "key.provider": { - selectedItem: "GCS: Google Cloud Storage", - }, - "key.provider.reader_impl": { - default: "gcsfs", - }, - "key.provider.service_account_json": {}, - "key.provider.storage": { - const: "GCS", - default: "GCS", - }, - "key.reader_options": {}, - "key.url": {}, - }); -}); diff --git a/airbyte-webapp/src/core/form/uiWidget.ts b/airbyte-webapp/src/core/form/uiWidget.ts deleted file mode 100644 index f3d6f695bafc..000000000000 --- a/airbyte-webapp/src/core/form/uiWidget.ts +++ /dev/null @@ -1,67 +0,0 @@ -import get from "lodash/get"; - -import { buildYupFormForJsonSchema } from "core/jsonSchema/schemaToYup"; -import { isDefined } from "utils/common"; - -import { FormBlock, WidgetConfigMap } from "./types"; - -export const buildPathInitialState = ( - formBlock: FormBlock[], - formValues: Record, - widgetState: WidgetConfigMap = {} -): Record => - formBlock.reduce((widgetStateBuilder, formItem) => { - switch (formItem._type) { - case "formGroup": - return buildPathInitialState(formItem.properties, formValues, widgetStateBuilder); - case "formItem": { - const resultObject: Record = {}; - - if (isDefined(formItem.const)) { - resultObject.const = formItem.const; - } - - if (isDefined(formItem.default)) { - resultObject.default = formItem.default; - } - - widgetStateBuilder[formItem.path] = resultObject; - return widgetStateBuilder; - } - case "formCondition": { - const defaultCondition = Object.entries(formItem.conditions).find(([key, subConditionItems]) => { - switch (subConditionItems._type) { - case "formGroup": { - const selectedValues = get(formValues, subConditionItems.path); - - const subPathSchema = buildYupFormForJsonSchema({ - type: "object", - ...subConditionItems.jsonSchema, - }); - - if (subPathSchema.isValidSync(selectedValues)) { - return key; - } - return null; - } - case "formItem": - return key; - } - - return null; - })?.[0]; - - const selectedPath = defaultCondition ?? Object.keys(formItem.conditions)?.[0]; - - widgetStateBuilder[formItem.path] = { - selectedItem: selectedPath, - }; - - if (formItem.conditions[selectedPath]) { - return buildPathInitialState([formItem.conditions[selectedPath]], formValues, widgetStateBuilder); - } - } - } - - return widgetStateBuilder; - }, widgetState); diff --git a/airbyte-webapp/src/core/jsonSchema/index.ts b/airbyte-webapp/src/core/jsonSchema/index.ts deleted file mode 100644 index abfe80b16dd1..000000000000 --- a/airbyte-webapp/src/core/jsonSchema/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./types"; -export * from "./schemaToUiWidget"; -export * from "./schemaToYup"; diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts b/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts deleted file mode 100644 index ca85400f3b69..000000000000 --- a/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import * as yup from "yup"; - -import { buildYupFormForJsonSchema } from "./schemaToYup"; -import { AirbyteJSONSchema } from "./types"; - -// Note: We have to check yup schema with JSON.stringify -// as exactly same objects throw now equality due to `Received: serializes to the same string` error -it("should build schema for simple case", () => { - const schema: AirbyteJSONSchema = { - type: "object", - title: "Postgres Source Spec", - required: ["host", "port", "user", "dbname", "is_field_no_default"], - properties: { - host: { type: "string", description: "Hostname of the database." }, - port: { - type: "integer", - maximum: 65536, - minimum: 0, - description: "Port of the database.", - }, - user: { - type: "string", - description: "Username to use to access the database.", - }, - is_sandbox: { - type: "boolean", - default: false, - }, - is_field_no_default: { - type: "boolean", - }, - dbname: { type: "string", description: "Name of the database." }, - password: { - type: "string", - description: "Password associated with the username.", - }, - reports: { - type: "array", - items: { - type: "string", - }, - }, - }, - additionalProperties: false, - }; - const yupSchema = buildYupFormForJsonSchema(schema); - - const expectedSchema = yup.object().shape({ - host: yup.string().trim().required("form.empty.error").transform(String), - port: yup - .number() - .min(0) - .max(65536) - .required("form.empty.error") - .transform((val) => (isNaN(val) ? undefined : val)), - user: yup.string().trim().required("form.empty.error").transform(String), - is_sandbox: yup.boolean().default(false), - is_field_no_default: yup.boolean().required("form.empty.error"), - dbname: yup.string().trim().required("form.empty.error").transform(String), - password: yup.string().trim().transform(String), - reports: yup.array().of(yup.string().trim().transform(String)), - }); - - expect(JSON.stringify(yupSchema)).toEqual(JSON.stringify(expectedSchema)); -}); - -it("should build schema for conditional case", () => { - const yupSchema = buildYupFormForJsonSchema( - { - type: "object", - required: ["start_date", "credentials"], - properties: { - start_date: { - type: "string", - }, - max_objects: { - type: "number", - }, - credentials: { - type: "object", - oneOf: [ - { - title: "api key", - required: ["api_key"], - properties: { - api_key: { - type: "string", - }, - }, - }, - { - title: "oauth", - required: ["redirect_uri"], - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, - ], - }, - }, - }, - { credentials: { selectedItem: "api key" } } - ); - - const expectedSchema = yup.object().shape({ - start_date: yup.string().trim().required("form.empty.error").transform(String), - max_objects: yup.number().transform((x) => x), - credentials: yup.object().shape({ - api_key: yup.string().trim().required("form.empty.error").transform(String), - }), - }); - - expect(JSON.stringify(yupSchema)).toEqual(JSON.stringify(expectedSchema)); -}); - -it("should build schema for conditional case with inner schema and selected uiwidget", () => { - const yupSchema = buildYupFormForJsonSchema( - { - type: "object", - properties: { - credentials: { - type: "object", - oneOf: [ - { - title: "api key", - required: ["api_key"], - properties: { - api_key: { - type: "string", - }, - }, - }, - { - title: "oauth", - required: ["redirect_uri"], - properties: { - redirect_uri: { - type: "string", - examples: ["https://api.hubspot.com/"], - }, - }, - }, - ], - }, - }, - }, - { "topKey.subKey.credentials": { selectedItem: "oauth" } }, - undefined, - "topKey", - "topKey.subKey" - ); - - const expectedSchema = yup.object().shape({ - credentials: yup.object().shape({ - redirect_uri: yup.string().trim().required("form.empty.error").transform(String), - }), - }); - - expect(JSON.stringify(yupSchema)).toEqual(JSON.stringify(expectedSchema)); -}); diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts b/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts deleted file mode 100644 index 433199089f67..000000000000 --- a/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { JSONSchema7 } from "json-schema"; -import * as yup from "yup"; - -import { WidgetConfigMap } from "core/form/types"; -import { isDefined } from "utils/common"; - -/** - * Returns yup.schema for validation - * - * This method builds yup schema based on jsonSchema ${@link JSONSchema7} and widgetConfig ${@link WidgetConfigMap}. - * Every property is walked through recursively in case it is condition | object | array. - * - * uiConfig is used to select currently selected oneOf conditions to build proper schema - * As uiConfig widget paths are .dot based (key1.innerModule1.innerModule2) propertyKey is provided recursively - * @param jsonSchema - * @param uiConfig uiConfig of widget currently selected in form - * @param parentSchema used in recursive schema building as required fields can be described in parentSchema - * @param propertyKey used in recursive schema building for building path for uiConfig - * @param propertyPath constructs path of property - */ -export const buildYupFormForJsonSchema = ( - jsonSchema: JSONSchema7, - uiConfig?: WidgetConfigMap, - parentSchema?: JSONSchema7, - propertyKey?: string, - propertyPath: string | undefined = propertyKey -): yup.AnySchema => { - let schema: - | yup.NumberSchema - | yup.StringSchema - | yup.AnyObjectSchema - | yup.ArraySchema - | yup.BooleanSchema - | null = null; - - if (jsonSchema.oneOf && uiConfig && propertyPath) { - let selectedSchema = jsonSchema.oneOf.find( - (condition) => typeof condition !== "boolean" && uiConfig[propertyPath]?.selectedItem === condition.title - ); - - // Select first oneOf path if no item selected - selectedSchema = selectedSchema ?? jsonSchema.oneOf[0]; - - if (selectedSchema && typeof selectedSchema !== "boolean") { - return buildYupFormForJsonSchema( - { type: jsonSchema.type, ...selectedSchema }, - uiConfig, - jsonSchema, - propertyKey, - propertyPath - ); - } - } - - switch (jsonSchema.type) { - case "string": - schema = yup - .string() - .transform((val) => String(val)) - .trim(); - - if (jsonSchema?.pattern !== undefined) { - schema = schema.matches(new RegExp(jsonSchema.pattern), "form.pattern.error"); - } - - break; - case "boolean": - schema = yup.boolean(); - break; - case "integer": - case "number": - schema = yup.number().transform((value) => (isNaN(value) ? undefined : value)); - - if (jsonSchema?.minimum !== undefined) { - schema = schema.min(jsonSchema?.minimum); - } - - if (jsonSchema?.maximum !== undefined) { - schema = schema.max(jsonSchema?.maximum); - } - break; - case "array": - if (typeof jsonSchema.items === "object" && !Array.isArray(jsonSchema.items)) { - schema = yup - .array() - .of( - buildYupFormForJsonSchema( - jsonSchema.items, - uiConfig, - jsonSchema, - propertyKey, - propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey - ) - ); - } - break; - case "object": - let objectSchema = yup.object(); - - const keyEntries = Object.entries(jsonSchema.properties || {}).map(([propertyKey, condition]) => [ - propertyKey, - typeof condition !== "boolean" - ? buildYupFormForJsonSchema( - condition, - uiConfig, - jsonSchema, - propertyKey, - propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey - ) - : yup.mixed(), - ]); - - if (keyEntries.length) { - objectSchema = objectSchema.shape(Object.fromEntries(keyEntries)); - } else { - objectSchema = objectSchema.default({}); - } - - schema = objectSchema; - } - - if (schema) { - const hasDefault = isDefined(jsonSchema.default); - - if (hasDefault) { - // @ts-expect-error can't infer correct type here so lets just use default from json_schema - schema = schema.default(jsonSchema.default); - } - - if (!hasDefault && jsonSchema.const) { - // @ts-expect-error can't infer correct type here so lets just use default from json_schema - schema = schema.oneOf([jsonSchema.const]).default(jsonSchema.const); - } - - if (jsonSchema.enum) { - // @ts-expect-error as enum is array we are going to use it as oneOf for yup - schema = schema.oneOf(jsonSchema.enum); - } - - const isRequired = - !hasDefault && - parentSchema && - Array.isArray(parentSchema?.required) && - parentSchema.required.find((item) => item === propertyKey); - - if (schema && isRequired) { - schema = schema.required("form.empty.error"); - } - } - - return schema || yup.mixed(); -}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index b38e04d3201f..ba74f8919b3e 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -187,6 +187,9 @@ "connectorForm.authenticate.required": "Authentication required", "connectorForm.reauthenticate": "Re-authenticate", "connectorForm.expandForm": "Expand this form to continue setting up your connector", + "connectorForm.error.oneOfWithNonObjects": "Spec uses oneOf without using object types for all conditions", + "connectorForm.error.oneOfWithoutConst": "Spec uses oneOf without a shared const property", + "connectorForm.error.topLevelNonObject": "Top level configuration has to be an object", "form.rawData": "Raw data (JSON)", "form.basicNormalization": "Normalized tabular data", @@ -595,6 +598,8 @@ "errorView.notFound": "Resource not found.", "errorView.notAuthorized": "You don’t have permission to access this page.", "errorView.title": "Oops! Something went wrong…", + "errorView.docLink": "Check out the documentation", + "errorView.upgradeConnectors": "Make sure your connectors are up to date", "errorView.retry": "Retry", "errorView.unknown": "Unknown", "errorView.unknownError": "Unknown error occurred", diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index 18b2aeacb8ac..9058ad193b10 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -170,8 +170,13 @@ export const ConnectorCard: React.FC (isEditMode && connector ? connector : { name: selectedConnectorDefinition?.name }), + [isEditMode, connector, selectedConnectorDefinition?.name] + ); return ( diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx index 0edfe7867991..27a9e27a60b2 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx @@ -6,7 +6,7 @@ import selectEvent from "react-select-event"; import { render, useMockIntersectionObserver } from "test-utils/testutils"; import { ConnectorDefinition } from "core/domain/connector"; -import { AirbyteJSONSchema } from "core/jsonSchema"; +import { AirbyteJSONSchema } from "core/jsonSchema/types"; import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; import { ConnectorForm, ConnectorFormProps } from "views/Connector/ConnectorForm"; @@ -97,6 +97,11 @@ const schema: AirbyteJSONSchema = { api_key: { type: "string", }, + type: { + type: "string", + const: "api", + default: "api", + }, }, }, { @@ -106,6 +111,11 @@ const schema: AirbyteJSONSchema = { type: "string", examples: ["https://api.hubspot.com/"], }, + type: { + type: "string", + const: "oauth", + default: "oauth", + }, }, }, ], @@ -215,11 +225,9 @@ describe("Service Form", () => { it("should display oneOf field", () => { const credentials = container.querySelector("div[data-testid='connectionConfiguration.credentials']"); - const credentialsValue = credentials?.querySelector("input[value='api key']"); const apiKey = container.querySelector("input[name='connectionConfiguration.credentials.api_key']"); expect(credentials).toBeInTheDocument(); expect(credentials?.getAttribute("role")).toEqual("combobox"); - expect(credentialsValue).toBeInTheDocument(); expect(apiKey).toBeInTheDocument(); }); @@ -294,7 +302,7 @@ describe("Service Form", () => { expect(result).toEqual({ name: "name", connectionConfiguration: { - credentials: { api_key: "test-api-key" }, + credentials: { api_key: "test-api-key", type: "api" }, emails: ["test@test.com"], host: "test-host", message: "test-message", @@ -329,7 +337,8 @@ describe("Service Form", () => { }); it("change oneOf field value", async () => { - const credentials = screen.getByTestId("connectionConfiguration.credentials"); + const apiKey = container.querySelector("input[name='connectionConfiguration.credentials.api_key']"); + expect(apiKey).toBeInTheDocument(); const selectContainer = getByTestId(container, "connectionConfiguration.credentials"); @@ -337,10 +346,7 @@ describe("Service Form", () => { container: document.body, }); - const credentialsValue = credentials.querySelector("input[value='oauth']"); const uri = container.querySelector("input[name='connectionConfiguration.credentials.redirect_uri']"); - - expect(credentialsValue).toBeInTheDocument(); expect(uri).toBeInTheDocument(); }); @@ -358,7 +364,7 @@ describe("Service Form", () => { await waitFor(() => userEvent.click(submit!)); expect(result.connectionConfiguration).toEqual({ - credentials: { redirect_uri: "test-uri" }, + credentials: { redirect_uri: "test-uri", type: "oauth" }, }); }); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx index a684eb6f5dfe..5e40157abfd7 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.tsx @@ -1,7 +1,5 @@ -import { Formik, getIn, setIn, useFormikContext } from "formik"; -import { JSONSchema7 } from "json-schema"; -import React, { useCallback, useEffect } from "react"; -import { useDeepCompareEffect } from "react-use"; +import { Formik } from "formik"; +import React, { useCallback } from "react"; import { FormChangeTracker } from "components/common/FormChangeTracker"; @@ -9,68 +7,11 @@ import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/doma import { FormikPatch } from "core/form/FormikPatch"; import { CheckConnectionRead } from "core/request/AirbyteClient"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; -import { isDefined } from "utils/common"; -import { ConnectorFormContextProvider, useConnectorForm } from "./connectorFormContext"; +import { ConnectorFormContextProvider } from "./connectorFormContext"; import { FormRoot } from "./FormRoot"; import { ConnectorCardValues, ConnectorFormValues } from "./types"; -import { useBuildForm, useBuildUiWidgetsContext, useConstructValidationSchema } from "./useBuildForm"; - -/** - * This function sets all initial const values in the form to current values - * @param schema - * @param initialValues - * @constructor - */ -const PatchInitialValuesWithWidgetConfig: React.FC<{ - schema: JSONSchema7; - initialValues: ConnectorFormValues; -}> = ({ schema, initialValues }) => { - const { widgetsInfo } = useConnectorForm(); - const { setFieldValue } = useFormikContext(); - - useDeepCompareEffect(() => { - const widgetsInfoEntries = Object.entries(widgetsInfo); - - // set all const fields to form field values, so we could send form - const patchedConstValues = widgetsInfoEntries - .filter(([_, value]) => isDefined(value.const)) - .reduce((acc, [key, value]) => setIn(acc, key, value.const), initialValues); - - // set default fields as current values, so values could be populated correctly - // fix for https://github.com/airbytehq/airbyte/issues/6791 - const patchedDefaultValues = widgetsInfoEntries - .filter(([key, value]) => isDefined(value.default) && !isDefined(getIn(patchedConstValues, key))) - .reduce((acc, [key, value]) => setIn(acc, key, value.default), patchedConstValues); - - if (patchedDefaultValues?.connectionConfiguration) { - setTimeout(() => { - // We need to push this out one execution slot, so the form isn't still in its - // initialization status and won't react to this call but would just take the initialValues instead. - setFieldValue("connectionConfiguration", patchedDefaultValues.connectionConfiguration); - }); - } - - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [schema]); - - return null; -}; - -/** - * Formik does not revalidate the form in case the validationSchema it's using changes. - * This component just forces a revalidation of the form whenever the validation schema changes. - */ -const RevalidateOnValidationSchemaChange: React.FC<{ validationSchema: unknown }> = ({ validationSchema }) => { - // The validationSchema is passed into this component instead of pulled from the FormikContext, since - // due to https://github.com/jaredpalmer/formik/issues/2092 the validationSchema from the formik context will - // always be undefined. - const { validateForm } = useFormikContext(); - useEffect(() => { - validateForm(); - }, [validateForm, validationSchema]); - return null; -}; +import { useBuildForm } from "./useBuildForm"; export interface ConnectorFormProps { formType: "source" | "destination"; @@ -107,16 +48,13 @@ export const ConnectorForm: React.FC = (props) => { connectorId, } = props; - const { formFields, initialValues, jsonSchema } = useBuildForm( + const { formFields, initialValues, validationSchema } = useBuildForm( + Boolean(isEditMode), formType, selectedConnectorDefinitionSpecification, formValues ); - const { uiWidgetsInfo, setUiWidgetsInfo, resetUiWidgetsInfo } = useBuildUiWidgetsContext(formFields, initialValues); - - const validationSchema = useConstructValidationSchema(jsonSchema, uiWidgetsInfo); - const getValues = useCallback( (values: ConnectorFormValues) => validationSchema.cast(values, { @@ -147,20 +85,15 @@ export const ConnectorForm: React.FC = (props) => { {({ dirty }) => ( - - = ({ formField, path, disabled }) => { - const { widgetsInfo, setUiWidgetsInfo } = useConnectorForm(); const { values, setValues } = useFormikContext(); const [, meta] = useField(path); - const currentlySelectedCondition = widgetsInfo[formField.path]?.selectedItem; + // the value at selectionPath determines which condition is selected + const currentSelectionValue = get(values, formField.selectionPath); + let currentlySelectedCondition: number | undefined = formField.selectionConstValues.indexOf(currentSelectionValue); + if (currentlySelectedCondition === -1) { + // there should always be a matching condition, but in some edge cases + // (e.g. breaking changes in specs) it's possible to have no matching value. + currentlySelectedCondition = undefined; + } const onOptionChange = useCallback( (selectedItem: DropDownOptionDataItem) => { @@ -45,19 +51,16 @@ export const ConditionSection: React.FC = ({ formField, p ) : values; - setUiWidgetsInfo(formField.path, { - selectedItem: selectedItem.value, - }); setValues(newValues); }, - [values, formField.conditions, setValues, setUiWidgetsInfo, formField.path] + [values, formField.conditions, setValues] ); const options = useMemo( () => - Object.keys(formField.conditions).map((dataItem) => ({ - label: dataItem, - value: dataItem, + formField.conditions.map((condition, index) => ({ + label: condition.title, + value: index, })), [formField.conditions] ); @@ -78,12 +81,15 @@ export const ConditionSection: React.FC = ({ formField, p /> } > - + {/* currentlySelectedCondition is only falsy if a malformed config is loaded which doesn't have a valid value for the const selection key. In this case, render the selection group as empty. */} + {typeof currentlySelectedCondition !== "undefined" && ( + + )} ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/PropertySection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/PropertySection.tsx index a7e333429670..14de86966708 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/PropertySection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/PropertySection.tsx @@ -6,7 +6,6 @@ import { LabeledSwitch } from "components"; import { FormBaseItem } from "core/form/types"; -import { useConnectorForm } from "../../connectorFormContext"; import { Control } from "../Property/Control"; import { PropertyError } from "../Property/PropertyError"; import { PropertyLabel } from "../Property/PropertyLabel"; @@ -22,12 +21,6 @@ const PropertySection: React.FC = ({ property, path, disab const propertyPath = path ?? property.path; const formikBag = useField(propertyPath); const [field, meta] = formikBag; - const { widgetsInfo } = useConnectorForm(); - - const overriddenComponent = widgetsInfo[propertyPath]?.component; - if (overriddenComponent) { - return <>{overriddenComponent(property, { disabled })}; - } const labelText = property.title || property.fieldKey; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx index 18932ca2670e..11d2f766536f 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/connectorFormContext.tsx @@ -3,15 +3,12 @@ import React, { useContext, useMemo } from "react"; import { AnySchema } from "yup"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; -import { WidgetConfigMap } from "core/form/types"; import { ConnectorFormValues } from "./types"; interface ConnectorFormContext { formType: "source" | "destination"; getValues: (values: ConnectorFormValues) => ConnectorFormValues; - widgetsInfo: WidgetConfigMap; - setUiWidgetsInfo: (path: string, value: Record) => void; resetConnectorForm: () => void; selectedConnectorDefinition: ConnectorDefinition; selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; @@ -32,9 +29,6 @@ export const useConnectorForm = (): ConnectorFormContext => { interface ConnectorFormContextProviderProps { selectedConnectorDefinition: ConnectorDefinition; - widgetsInfo: WidgetConfigMap; - setUiWidgetsInfo: (path: string, value: Record) => void; - resetUiWidgetsInfo: () => void; formType: "source" | "destination"; isEditMode?: boolean; getValues: (values: ConnectorFormValues) => ConnectorFormValues; @@ -46,9 +40,6 @@ interface ConnectorFormContextProviderProps { export const ConnectorFormContextProvider: React.FC> = ({ selectedConnectorDefinition, children, - widgetsInfo, - setUiWidgetsInfo, - resetUiWidgetsInfo, selectedConnectorDefinitionSpecification, getValues, formType, @@ -60,9 +51,7 @@ export const ConnectorFormContextProvider: React.FC(() => { const context: ConnectorFormContext = { - widgetsInfo, getValues, - setUiWidgetsInfo, selectedConnectorDefinition, selectedConnectorDefinitionSpecification, formType, @@ -71,14 +60,11 @@ export const ConnectorFormContextProvider: React.FC { resetForm(); - resetUiWidgetsInfo(); }, }; return context; }, [ - widgetsInfo, getValues, - setUiWidgetsInfo, selectedConnectorDefinition, selectedConnectorDefinitionSpecification, formType, @@ -86,7 +72,6 @@ export const ConnectorFormContextProvider: React.FC{children}; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx index f36a2e9d9f5a..fdaaa82cbbb6 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useBuildForm.tsx @@ -1,103 +1,105 @@ import { JSONSchema7, JSONSchema7Definition } from "json-schema"; -import merge from "lodash/merge"; -import { useCallback, useMemo, useState } from "react"; +import { useMemo } from "react"; import { useIntl } from "react-intl"; import { AnySchema } from "yup"; -import { ConnectorDefinitionSpecification } from "core/domain/connector"; -import { FormBlock, WidgetConfig, WidgetConfigMap } from "core/form/types"; -import { buildPathInitialState } from "core/form/uiWidget"; -import { jsonSchemaToUiWidget } from "core/jsonSchema/schemaToUiWidget"; -import { buildYupFormForJsonSchema } from "core/jsonSchema/schemaToYup"; +import { ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; +import { FormBuildError, isFormBuildError } from "core/form/FormBuildError"; +import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; +import { buildYupFormForJsonSchema } from "core/form/schemaToYup"; +import { FormBlock, FormGroupItem } from "core/form/types"; import { ConnectorFormValues } from "./types"; export interface BuildFormHook { initialValues: ConnectorFormValues; formFields: FormBlock; - jsonSchema: JSONSchema7; + validationSchema: AnySchema; +} + +function setDefaultValues(formGroup: FormGroupItem, values: Record) { + formGroup.properties.forEach((property) => { + if (property.const) { + values[property.fieldKey] = property.const; + } + if (property.default) { + values[property.fieldKey] = property.default; + } + switch (property._type) { + case "formGroup": + values[property.fieldKey] = {}; + setDefaultValues(property, values[property.fieldKey] as Record); + break; + case "formCondition": + // implicitly select the first option (do not respect a potential default value) + values[property.fieldKey] = {}; + setDefaultValues(property.conditions[0], values[property.fieldKey] as Record); + } + }); } export function useBuildForm( + isEditMode: boolean, formType: "source" | "destination", selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification, initialValues?: Partial ): BuildFormHook { const { formatMessage } = useIntl(); - const jsonSchema: JSONSchema7 = useMemo( - () => ({ - type: "object", - properties: { - name: { - type: "string", - title: formatMessage({ id: `form.${formType}Name` }), - description: formatMessage({ id: `form.${formType}Name.message` }), + try { + const jsonSchema: JSONSchema7 = useMemo( + () => ({ + type: "object", + properties: { + name: { + type: "string", + title: formatMessage({ id: `form.${formType}Name` }), + description: formatMessage({ id: `form.${formType}Name.message` }), + }, + connectionConfiguration: + selectedConnectorDefinitionSpecification.connectionSpecification as JSONSchema7Definition, }, - connectionConfiguration: - selectedConnectorDefinitionSpecification.connectionSpecification as JSONSchema7Definition, - }, - required: ["name"], - }), - [formType, formatMessage, selectedConnectorDefinitionSpecification.connectionSpecification] - ); - const startValues = useMemo( - () => ({ - name: "", - connectionConfiguration: {}, - ...initialValues, - }), - [initialValues] - ); - - const formFields = useMemo(() => jsonSchemaToUiWidget(jsonSchema), [jsonSchema]); - - return { - initialValues: startValues, - formFields, - jsonSchema, - }; -} - -// useBuildUiWidgetsContext hook -interface BuildUiWidgetsContextHook { - uiWidgetsInfo: WidgetConfigMap; - setUiWidgetsInfo: (widgetId: string, updatedValues: WidgetConfig) => void; - resetUiWidgetsInfo: () => void; -} + required: ["name"], + }), + [formType, formatMessage, selectedConnectorDefinitionSpecification.connectionSpecification] + ); -export const useBuildUiWidgetsContext = ( - formFields: FormBlock[] | FormBlock, - formValues: ConnectorFormValues -): BuildUiWidgetsContextHook => { - const [overriddenWidgetState, setUiWidgetsInfo] = useState({}); + const formFields = useMemo(() => jsonSchemaToFormBlock(jsonSchema), [jsonSchema]); - // As schema is dynamic, it is possible, that new updated values, will differ from one stored. - const mergedState = useMemo( - () => - merge( - buildPathInitialState(Array.isArray(formFields) ? formFields : [formFields], formValues), - merge(overriddenWidgetState) - ), - [formFields, formValues, overriddenWidgetState] - ); + if (formFields._type !== "formGroup") { + throw new FormBuildError("connectorForm.error.topLevelNonObject"); + } - const setUiWidgetsInfoSubState = useCallback( - (widgetId: string, updatedValues: WidgetConfig) => setUiWidgetsInfo({ ...mergedState, [widgetId]: updatedValues }), - [mergedState, setUiWidgetsInfo] - ); + const startValues = useMemo(() => { + if (isEditMode) { + return { + name: "", + connectionConfiguration: {}, + ...initialValues, + }; + } + const baseValues = { + name: "", + connectionConfiguration: {}, + ...initialValues, + }; - const resetUiWidgetsInfo = useCallback(() => { - setUiWidgetsInfo({}); - }, []); + setDefaultValues(formFields, baseValues as Record); - return { - uiWidgetsInfo: mergedState, - setUiWidgetsInfo: setUiWidgetsInfoSubState, - resetUiWidgetsInfo, - }; -}; + return baseValues; + }, [formFields, initialValues, isEditMode]); -// As validation schema depends on what path of oneOf is currently selected in jsonschema -export const useConstructValidationSchema = (jsonSchema: JSONSchema7, uiWidgetsInfo: WidgetConfigMap): AnySchema => - useMemo(() => buildYupFormForJsonSchema(jsonSchema, uiWidgetsInfo), [uiWidgetsInfo, jsonSchema]); + const validationSchema = useMemo(() => buildYupFormForJsonSchema(jsonSchema, formFields), [formFields, jsonSchema]); + return { + initialValues: startValues, + formFields, + validationSchema, + }; + } catch (e) { + // catch and re-throw form-build errors to enrich them with the connector id + if (isFormBuildError(e)) { + throw new FormBuildError(e.message, ConnectorSpecification.id(selectedConnectorDefinitionSpecification)); + } + throw e; + } +} diff --git a/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx b/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx index 316b7e33ebcf..bfb7b36d6bd5 100644 --- a/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx +++ b/airbyte-webapp/src/views/common/ErrorOccurredView/ErrorOccurredView.tsx @@ -8,11 +8,20 @@ import styles from "./ErrorOccurredView.module.scss"; interface ErrorOccurredViewProps { message: React.ReactNode; + /** + * URL to relevant documentation for the error if available + */ + docLink?: string; ctaButtonText?: React.ReactNode; onCtaButtonClick?: React.MouseEventHandler; } -export const ErrorOccurredView: React.FC = ({ message, onCtaButtonClick, ctaButtonText }) => { +export const ErrorOccurredView: React.FC = ({ + message, + onCtaButtonClick, + ctaButtonText, + docLink, +}) => { return (
@@ -21,6 +30,13 @@ export const ErrorOccurredView: React.FC = ({ message, o

{message}

+ {docLink && ( +

+ + + +

+ )} {onCtaButtonClick && ctaButtonText && (