diff --git a/airbyte-webapp/.gitignore b/airbyte-webapp/.gitignore index ffcb7d3c3a8c..8e572d98ba49 100644 --- a/airbyte-webapp/.gitignore +++ b/airbyte-webapp/.gitignore @@ -29,5 +29,6 @@ yarn-error.log* storybook-static/ -# Ignore generated API client, since it's automatically generated +# Ignore generated API clients, since they're automatically generated /src/core/request/AirbyteClient.ts +/src/core/request/ConnectorBuilderClient.ts diff --git a/airbyte-webapp/orval.config.ts b/airbyte-webapp/orval.config.ts index d75bda5620e4..daac7be19c0c 100644 --- a/airbyte-webapp/orval.config.ts +++ b/airbyte-webapp/orval.config.ts @@ -22,4 +22,25 @@ export default defineConfig({ }, }, }, + connectorBuilder: { + input: "../connector-builder-server/src/main/openapi/openapi.yaml", + output: { + target: "./src/core/request/ConnectorBuilderClient.ts", + prettier: true, + override: { + header: (info) => [ + `eslint-disable`, + `Generated by orval 🍺`, + `Do not edit manually. Run "npm run generate-client" instead.`, + ...(info.title ? [info.title] : []), + ...(info.description ? [info.description] : []), + ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []), + ], + mutator: { + path: "./src/core/request/apiOverride.ts", + name: "apiOverride", + }, + }, + }, + }, }); diff --git a/airbyte-webapp/src/config/defaultConfig.ts b/airbyte-webapp/src/config/defaultConfig.ts index c01720a36a4b..e4225db9a819 100644 --- a/airbyte-webapp/src/config/defaultConfig.ts +++ b/airbyte-webapp/src/config/defaultConfig.ts @@ -5,6 +5,7 @@ const defaultConfig: Config = { healthCheckInterval: 20000, version: "dev", apiUrl: `${window.location.protocol}//${window.location.hostname}:8001/api`, + connectorBuilderApiUrl: `${window.location.protocol}//${window.location.hostname}:8080/`, integrationUrl: "/docs", oauthRedirectUrl: `${window.location.protocol}//${window.location.host}`, }; diff --git a/airbyte-webapp/src/config/types.ts b/airbyte-webapp/src/config/types.ts index ad818fd46446..62a0ab434e33 100644 --- a/airbyte-webapp/src/config/types.ts +++ b/airbyte-webapp/src/config/types.ts @@ -21,6 +21,7 @@ declare global { export interface Config { segment: { token: string; enabled: boolean }; apiUrl: string; + connectorBuilderApiUrl: string; oauthRedirectUrl: string; healthCheckInterval: number; version?: string; diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts new file mode 100644 index 000000000000..283df6e8f1c1 --- /dev/null +++ b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts @@ -0,0 +1,82 @@ +import { + StreamRead, + StreamReadRequestBody, + StreamsListRead, + StreamsListRequestBody, +} from "core/request/ConnectorBuilderClient"; + +import { AirbyteRequestService } from "../../request/AirbyteRequestService"; + +export class ConnectorBuilderRequestService extends AirbyteRequestService { + public readStream(readParams: StreamReadRequestBody): Promise { + // TODO: uncomment this and remove mock responses once there is a real API to call + // return readStream(readParams, this.requestOptions); + console.log("------------"); + console.log(`Stream: ${readParams.stream}`); + console.log(`Connector manifest:\n${JSON.stringify(readParams.manifest)}`); + console.log(`Config:\n${JSON.stringify(readParams.config)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + logs: [ + { level: "INFO", message: "Syncing stream: rates " }, + { level: "INFO", message: "Setting state of rates stream to {'date': '2022-09-25'}" }, + ], + slices: [ + { + sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" }, + state: { + type: "STREAM", + stream: { stream_descriptor: { name: readParams.stream }, stream_state: { date: "2022-09-26" } }, + data: { rates: { date: "2022-09-26" } }, + }, + pages: [ + { + records: [ + { + stream: readParams.stream, + data: { + id: "dp_123", + object: readParams.stream, + amount: 2000, + balance_transaction: "txn_123", + }, + }, + ], + request: { + url: "https://api.com/path", + }, + response: { + status: 200, + }, + }, + ], + }, + ], + }; + }); + } + + public listStreams(listParams: StreamsListRequestBody): Promise { + // TODO: uncomment this and remove mock responses once there is a real API to call + // return listStreams(listParams, this.requestOptions); + console.log(`Received listStreams body: ${JSON.stringify(listParams)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + streams: [ + { + name: "disputes", + url: "https://api.com/disputes", + }, + { + name: "transactions", + url: "https://api.com/transactions", + }, + { + name: "users", + url: "https://api.com/users", + }, + ], + }; + }); + } +} diff --git a/airbyte-webapp/src/pages/connector-builder/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx similarity index 100% rename from airbyte-webapp/src/pages/connector-builder/ConnectorBuilderPage.tsx rename to airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 1010703eba2c..ba76c4c73df1 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -15,7 +15,7 @@ import MainView from "views/layout/MainView"; import { WorkspaceRead } from "../core/request/AirbyteClient"; import ConnectionPage from "./ConnectionPage"; -import { ConnectorBuilderPage } from "./connector-builder/ConnectorBuilderPage"; +import { ConnectorBuilderPage } from "./ConnectorBuilderPage/ConnectorBuilderPage"; import DestinationPage from "./DestinationPage"; import OnboardingPage from "./OnboardingPage"; import PreferencesPage from "./PreferencesPage"; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts new file mode 100644 index 000000000000..c1fb27b0e745 --- /dev/null +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts @@ -0,0 +1,45 @@ +import { useQuery } from "react-query"; + +import { useConfig } from "config"; +import { ConnectorBuilderRequestService } from "core/domain/connectorBuilder/ConnectorBuilderRequestService"; +import { + StreamReadRequestBody, + StreamReadRequestBodyConfig, + StreamReadRequestBodyManifest, + StreamsListRequestBody, +} from "core/request/ConnectorBuilderClient"; +import { useSuspenseQuery } from "services/connector/useSuspenseQuery"; +import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; +import { useInitService } from "services/useInitService"; + +const connectorBuilderKeys = { + all: ["connectorBuilder"] as const, + read: (streamName: string, manifest: StreamReadRequestBodyManifest, config: StreamReadRequestBodyConfig) => + [...connectorBuilderKeys.all, "read", { streamName, manifest, config }] as const, + list: (manifest: StreamReadRequestBodyManifest) => [...connectorBuilderKeys.all, "list", { manifest }] as const, +}; + +function useConnectorBuilderService() { + const config = useConfig(); + const middlewares = useDefaultRequestMiddlewares(); + return useInitService( + () => new ConnectorBuilderRequestService(config.connectorBuilderApiUrl, middlewares), + [config.connectorBuilderApiUrl, middlewares] + ); +} + +export const useReadStream = (params: StreamReadRequestBody) => { + const service = useConnectorBuilderService(); + + return useQuery( + connectorBuilderKeys.read(params.stream, params.manifest, params.config), + () => service.readStream(params), + { refetchOnWindowFocus: false, enabled: false } + ); +}; + +export const useListStreams = (params: StreamsListRequestBody) => { + const service = useConnectorBuilderService(); + + return useSuspenseQuery(connectorBuilderKeys.list(params.manifest), () => service.listStreams(params)); +}; diff --git a/connector-builder-server/src/main/openapi/openapi.yaml b/connector-builder-server/src/main/openapi/openapi.yaml index 5479827c5e15..9a6145e83989 100644 --- a/connector-builder-server/src/main/openapi/openapi.yaml +++ b/connector-builder-server/src/main/openapi/openapi.yaml @@ -18,6 +18,7 @@ paths: /v1/stream/read: post: summary: Reads a specific stream in the source. TODO in a later phase - only read a single slice of data. + operationId: readStream requestBody: content: application/json: @@ -37,7 +38,8 @@ paths: $ref: "#/components/responses/InvalidInputResponse" /v1/streams/list: post: - summary: List all streams present in the connector definition, along with their specific request URLs + summary: List all streams present in the connector manifest, along with their specific request URLs + operationId: listStreams requestBody: content: application/json: @@ -61,53 +63,71 @@ components: StreamRead: type: object required: + - logs - slices properties: + logs: + type: array + description: The LOG AirbyteMessages that were emitted during the read of this slice + items: + type: object + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteLogMessage" slices: type: array - description: The stream slices returned from the read command + description: The stream slices returned from the read command. If no stream slicer is configured, this should contain a single item containing all of the results. items: type: object required: - - sliceDescriptor - pages properties: - sliceDescriptor: - type: object - description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}' pages: type: array - description: The pages returned from the read command + description: The pages returned from the read command. If no pagination is configured, this should contain a single item containing all of the results. items: type: object required: - - airbyteMessages + - records - request - response properties: - airbyteMessages: + records: type: array - description: The RECORD/STATE/LOG AirbyteMessages coming from the read operation for this page + description: The RECORD AirbyteMessages coming from the read operation for this page items: - $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteMessage" + type: object + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteRecordMessage" request: $ref: "#/components/schemas/HttpRequest" response: $ref: "#/components/schemas/HttpResponse" + sliceDescriptor: + type: object + description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}. This can be omitted if a stream slicer is not configured.' + state: + type: object + description: The STATE AirbyteMessage emitted at the end of this slice. This can be omitted if a stream slicer is not configured. + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" StreamReadRequestBody: type: object required: - - definition + - manifest - stream + - config properties: - definition: - $ref: "#/components/schemas/ConnectorDefinitionBody" - description: The config-based connector definition contents + manifest: + type: object + description: The config-based connector manifest contents + # $ref: "#/components/schemas/ConnectorManifest" stream: type: string description: Name of the stream to read + config: + type: object + description: The config blob containing the user inputs for testing state: - $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" + type: object + description: The AirbyteStateMessage object to use as the starting state for this read + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" # --- Potential addition for a later phase --- # numPages: # type: integer @@ -144,18 +164,20 @@ components: headers: type: object description: The headers of the HTTP response, if any - ConnectorDefinitionBody: - $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json - AirbyteProtocol: - $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml + # --- Commenting out for now since they do not work with our orval openapi client generator --- + # ConnectorManifest: + # $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json + # AirbyteProtocol: + # $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml StreamsListRequestBody: type: object required: - - definition + - manifest properties: - definition: - $ref: "#/components/schemas/ConnectorDefinitionBody" - description: The config-based connector definition contents + manifest: + type: object + description: The config-based connector manifest contents + # $ref: "#/components/schemas/ConnectorManifest" StreamsListRead: type: object required: @@ -165,7 +187,7 @@ components: type: array items: type: object - description: The stream names present in the connector definition + description: The stream names present in the connector manifest required: - name - url