From 28cccb2346c2a13b22cf7e4398ea1928263cee74 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 20 Oct 2022 14:21:55 -0700 Subject: [PATCH 1/9] update openapi for connector builder to work with orval for now --- .../src/main/openapi/openapi.yaml | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/connector-builder-server/src/main/openapi/openapi.yaml b/connector-builder-server/src/main/openapi/openapi.yaml index 5479827c5e15..7d215a4d6678 100644 --- a/connector-builder-server/src/main/openapi/openapi.yaml +++ b/connector-builder-server/src/main/openapi/openapi.yaml @@ -18,6 +18,7 @@ paths: /v1/stream/read: post: summary: Reads a specific stream in the source. TODO in a later phase - only read a single slice of data. + operationId: readStream requestBody: content: application/json: @@ -38,6 +39,7 @@ paths: /v1/streams/list: post: summary: List all streams present in the connector definition, along with their specific request URLs + operationId: listStreams requestBody: content: application/json: @@ -89,7 +91,8 @@ components: type: array description: The RECORD/STATE/LOG AirbyteMessages coming from the read operation for this page items: - $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteMessage" + type: object + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteMessage" request: $ref: "#/components/schemas/HttpRequest" response: @@ -97,17 +100,24 @@ components: StreamReadRequestBody: type: object required: - - definition + - connectorDefinition - stream + - config properties: - definition: - $ref: "#/components/schemas/ConnectorDefinitionBody" + connectorDefinition: + type: object description: The config-based connector definition contents + # $ref: "#/components/schemas/ConnectorDefinitionBody" stream: type: string description: Name of the stream to read state: - $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" + type: object + description: The AirbyteStateMessage object to use as the starting state for this read + config: + type: object + description: The config blob containing the user inputs for testing + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" # --- Potential addition for a later phase --- # numPages: # type: integer @@ -144,18 +154,20 @@ components: headers: type: object description: The headers of the HTTP response, if any - ConnectorDefinitionBody: - $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json - AirbyteProtocol: - $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml + # --- Commenting out for now since they do not work with our orval openapi client generator --- + # ConnectorDefinitionBody: + # $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json + # AirbyteProtocol: + # $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml StreamsListRequestBody: type: object required: - - definition + - connectorDefinition properties: - definition: - $ref: "#/components/schemas/ConnectorDefinitionBody" + connectorDefinition: + type: object description: The config-based connector definition contents + # $ref: "#/components/schemas/ConnectorDefinitionBody" StreamsListRead: type: object required: From 2366d0b0f86c0b3e8a76030b14f99b28b2b0ecb4 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 20 Oct 2022 14:22:35 -0700 Subject: [PATCH 2/9] add orval configuration for generating the connector builder client --- airbyte-webapp/.gitignore | 3 ++- airbyte-webapp/orval.config.ts | 21 +++++++++++++++++++++ airbyte-webapp/src/config/defaultConfig.ts | 1 + airbyte-webapp/src/config/types.ts | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/.gitignore b/airbyte-webapp/.gitignore index ffcb7d3c3a8c..8e572d98ba49 100644 --- a/airbyte-webapp/.gitignore +++ b/airbyte-webapp/.gitignore @@ -29,5 +29,6 @@ yarn-error.log* storybook-static/ -# Ignore generated API client, since it's automatically generated +# Ignore generated API clients, since they're automatically generated /src/core/request/AirbyteClient.ts +/src/core/request/ConnectorBuilderClient.ts diff --git a/airbyte-webapp/orval.config.ts b/airbyte-webapp/orval.config.ts index d75bda5620e4..daac7be19c0c 100644 --- a/airbyte-webapp/orval.config.ts +++ b/airbyte-webapp/orval.config.ts @@ -22,4 +22,25 @@ export default defineConfig({ }, }, }, + connectorBuilder: { + input: "../connector-builder-server/src/main/openapi/openapi.yaml", + output: { + target: "./src/core/request/ConnectorBuilderClient.ts", + prettier: true, + override: { + header: (info) => [ + `eslint-disable`, + `Generated by orval 🍺`, + `Do not edit manually. Run "npm run generate-client" instead.`, + ...(info.title ? [info.title] : []), + ...(info.description ? [info.description] : []), + ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []), + ], + mutator: { + path: "./src/core/request/apiOverride.ts", + name: "apiOverride", + }, + }, + }, + }, }); diff --git a/airbyte-webapp/src/config/defaultConfig.ts b/airbyte-webapp/src/config/defaultConfig.ts index c01720a36a4b..e4225db9a819 100644 --- a/airbyte-webapp/src/config/defaultConfig.ts +++ b/airbyte-webapp/src/config/defaultConfig.ts @@ -5,6 +5,7 @@ const defaultConfig: Config = { healthCheckInterval: 20000, version: "dev", apiUrl: `${window.location.protocol}//${window.location.hostname}:8001/api`, + connectorBuilderApiUrl: `${window.location.protocol}//${window.location.hostname}:8080/`, integrationUrl: "/docs", oauthRedirectUrl: `${window.location.protocol}//${window.location.host}`, }; diff --git a/airbyte-webapp/src/config/types.ts b/airbyte-webapp/src/config/types.ts index ad818fd46446..62a0ab434e33 100644 --- a/airbyte-webapp/src/config/types.ts +++ b/airbyte-webapp/src/config/types.ts @@ -21,6 +21,7 @@ declare global { export interface Config { segment: { token: string; enabled: boolean }; apiUrl: string; + connectorBuilderApiUrl: string; oauthRedirectUrl: string; healthCheckInterval: number; version?: string; From 8c5ed505d9a137c9b4aa0ebb1cd51076707919d2 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 20 Oct 2022 14:24:58 -0700 Subject: [PATCH 3/9] create ConnectorBuilderApiService with methods to call the generated connector builder API client code --- .../ConnectorBuilderApiService.ts | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts diff --git a/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts b/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts new file mode 100644 index 000000000000..6cc7569e346c --- /dev/null +++ b/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts @@ -0,0 +1,157 @@ +import { useQuery } from "react-query"; + +import { useConfig } from "config"; +import { AirbyteRequestService } from "core/request/AirbyteRequestService"; +import { + StreamRead, + StreamReadRequestBody, + StreamReadRequestBodyConfig, + StreamReadRequestBodyConnectorDefinition, + StreamsListRead, + StreamsListRequestBody, +} from "core/request/ConnectorBuilderClient"; +import { useSuspenseQuery } from "services/connector/useSuspenseQuery"; +import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; +import { useInitService } from "services/useInitService"; + +class ConnectorBuilderService extends AirbyteRequestService { + public readStream(body: StreamReadRequestBody): Promise { + // TODO: uncomment this once there is a real API to call + // return readStream(body, this.requestOptions); + console.log("------------"); + console.log(`Stream: ${body.stream}`); + console.log(`Connector definition:\n${JSON.stringify(body.connectorDefinition)}`); + console.log(`Config:\n${JSON.stringify(body.config)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + slices: [ + { + sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" }, + pages: [ + { + airbyteMessages: [ + { + type: "RECORD", + record: { + stream: body.stream, + data: { + id: "dp_123", + object: body.stream, + amount: 2000, + balance_transaction: "txn_123", + }, + }, + }, + { + type: "STATE", + state: { + data: { + timestamp: "2022-10-20T02:00:59Z", + }, + }, + }, + ], + request: { + url: "https://api.com/path", + }, + response: { + status: 200, + }, + }, + { + airbyteMessages: [ + { + type: "RECORD", + record: { + stream: body.stream, + data: { + id: "dp_123", + object: body.stream, + amount: 2000, + balance_transaction: "txn_123", + }, + }, + }, + { + type: "STATE", + state: { + data: { + timestamp: "2022-10-20T02:00:59Z", + }, + }, + }, + ], + request: { + url: "https://api.com/path", + }, + response: { + status: 200, + }, + }, + ], + }, + ], + }; + }); + } + + public listStreams(body: StreamsListRequestBody): Promise { + // TODO: uncomment this once there is a real API to call + // return listStreams(body, this.requestOptions); + console.log(`Received listStreams body: ${JSON.stringify(body)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + streams: [ + { + name: "disputes", + url: "https://api.com/disputes", + }, + { + name: "transactions", + url: "https://api.com/transactions", + }, + { + name: "users", + url: "https://api.com/users", + }, + ], + }; + }); + } +} + +const connectorBuilderKeys = { + all: ["connectorBuilder"] as const, + read: ( + streamName: string, + connectorDefinition: StreamReadRequestBodyConnectorDefinition, + config: StreamReadRequestBodyConfig + ) => [...connectorBuilderKeys.all, "read", { streamName, connectorDefinition, config }] as const, + list: (connectorDefinition: StreamReadRequestBodyConnectorDefinition) => + [...connectorBuilderKeys.all, "list", { connectorDefinition }] as const, +}; + +function useConnectorBuilderService() { + const config = useConfig(); + const middlewares = useDefaultRequestMiddlewares(); + return useInitService( + () => new ConnectorBuilderService(config.connectorBuilderApiUrl, middlewares), + [config.connectorBuilderApiUrl, middlewares] + ); +} + +export const useReadStream = (params: StreamReadRequestBody) => { + const service = useConnectorBuilderService(); + + return useQuery( + connectorBuilderKeys.read(params.stream, params.connectorDefinition, params.config), + () => service.readStream(params), + { refetchOnWindowFocus: false, enabled: false } + ); +}; + +export const useListStreams = (params: StreamsListRequestBody) => { + const service = useConnectorBuilderService(); + + return useSuspenseQuery(connectorBuilderKeys.list(params.connectorDefinition), () => service.listStreams(params)); +}; From 2512bdd3d1ee2698483efce10d10a0db5ad2b9bc Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 20 Oct 2022 14:39:02 -0700 Subject: [PATCH 4/9] rename connector definition to manifest --- .../ConnectorBuilderApiService.ts | 16 +++++------ .../src/main/openapi/openapi.yaml | 28 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts b/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts index 6cc7569e346c..69f0f9ca3f3d 100644 --- a/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts +++ b/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts @@ -6,7 +6,7 @@ import { StreamRead, StreamReadRequestBody, StreamReadRequestBodyConfig, - StreamReadRequestBodyConnectorDefinition, + StreamReadRequestBodyManifest, StreamsListRead, StreamsListRequestBody, } from "core/request/ConnectorBuilderClient"; @@ -20,7 +20,7 @@ class ConnectorBuilderService extends AirbyteRequestService { // return readStream(body, this.requestOptions); console.log("------------"); console.log(`Stream: ${body.stream}`); - console.log(`Connector definition:\n${JSON.stringify(body.connectorDefinition)}`); + console.log(`Connector manifest:\n${JSON.stringify(body.manifest)}`); console.log(`Config:\n${JSON.stringify(body.config)}`); return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { return { @@ -124,11 +124,11 @@ const connectorBuilderKeys = { all: ["connectorBuilder"] as const, read: ( streamName: string, - connectorDefinition: StreamReadRequestBodyConnectorDefinition, + manifest: StreamReadRequestBodyManifest, config: StreamReadRequestBodyConfig - ) => [...connectorBuilderKeys.all, "read", { streamName, connectorDefinition, config }] as const, - list: (connectorDefinition: StreamReadRequestBodyConnectorDefinition) => - [...connectorBuilderKeys.all, "list", { connectorDefinition }] as const, + ) => [...connectorBuilderKeys.all, "read", { streamName, manifest, config }] as const, + list: (manifest: StreamReadRequestBodyManifest) => + [...connectorBuilderKeys.all, "list", { manifest }] as const, }; function useConnectorBuilderService() { @@ -144,7 +144,7 @@ export const useReadStream = (params: StreamReadRequestBody) => { const service = useConnectorBuilderService(); return useQuery( - connectorBuilderKeys.read(params.stream, params.connectorDefinition, params.config), + connectorBuilderKeys.read(params.stream, params.manifest, params.config), () => service.readStream(params), { refetchOnWindowFocus: false, enabled: false } ); @@ -153,5 +153,5 @@ export const useReadStream = (params: StreamReadRequestBody) => { export const useListStreams = (params: StreamsListRequestBody) => { const service = useConnectorBuilderService(); - return useSuspenseQuery(connectorBuilderKeys.list(params.connectorDefinition), () => service.listStreams(params)); + return useSuspenseQuery(connectorBuilderKeys.list(params.manifest), () => service.listStreams(params)); }; diff --git a/connector-builder-server/src/main/openapi/openapi.yaml b/connector-builder-server/src/main/openapi/openapi.yaml index 7d215a4d6678..19ce1ddd4bfd 100644 --- a/connector-builder-server/src/main/openapi/openapi.yaml +++ b/connector-builder-server/src/main/openapi/openapi.yaml @@ -38,7 +38,7 @@ paths: $ref: "#/components/responses/InvalidInputResponse" /v1/streams/list: post: - summary: List all streams present in the connector definition, along with their specific request URLs + summary: List all streams present in the connector manifest, along with their specific request URLs operationId: listStreams requestBody: content: @@ -100,23 +100,23 @@ components: StreamReadRequestBody: type: object required: - - connectorDefinition + - manifest - stream - config properties: - connectorDefinition: + manifest: type: object - description: The config-based connector definition contents - # $ref: "#/components/schemas/ConnectorDefinitionBody" + description: The config-based connector manifest contents + # $ref: "#/components/schemas/ConnectorManifest" stream: type: string description: Name of the stream to read - state: - type: object - description: The AirbyteStateMessage object to use as the starting state for this read config: type: object description: The config blob containing the user inputs for testing + state: + type: object + description: The AirbyteStateMessage object to use as the starting state for this read # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" # --- Potential addition for a later phase --- # numPages: @@ -155,19 +155,19 @@ components: type: object description: The headers of the HTTP response, if any # --- Commenting out for now since they do not work with our orval openapi client generator --- - # ConnectorDefinitionBody: + # ConnectorManifest: # $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json # AirbyteProtocol: # $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml StreamsListRequestBody: type: object required: - - connectorDefinition + - manifest properties: - connectorDefinition: + manifest: type: object - description: The config-based connector definition contents - # $ref: "#/components/schemas/ConnectorDefinitionBody" + description: The config-based connector manifest contents + # $ref: "#/components/schemas/ConnectorManifest" StreamsListRead: type: object required: @@ -177,7 +177,7 @@ components: type: array items: type: object - description: The stream names present in the connector definition + description: The stream names present in the connector manifest required: - name - url From 5635349839923a99d41a1dc92b5471503d307441 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 21 Oct 2022 13:44:22 -0700 Subject: [PATCH 5/9] refactor builder api service to match existing patterns --- .../ConnectorBuilderRequestService.ts | 109 ++++++++++++ .../ConnectorBuilderApiService.ts | 157 ------------------ .../ConnectorBuilderApiService.ts | 45 +++++ 3 files changed, 154 insertions(+), 157 deletions(-) create mode 100644 airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts delete mode 100644 airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts create mode 100644 airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts new file mode 100644 index 000000000000..18a997e31986 --- /dev/null +++ b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts @@ -0,0 +1,109 @@ +import { StreamReadRequestBody, StreamsListRequestBody } from "core/request/ConnectorBuilderClient"; + +import { AirbyteRequestService } from "../../request/AirbyteRequestService"; + +export class ConnectorBuilderRequestService extends AirbyteRequestService { + public readStream(readParams: StreamReadRequestBody) { + // TODO: uncomment this once there is a real API to call + // return readStream(readParams, this.requestOptions); + console.log("------------"); + console.log(`Stream: ${readParams.stream}`); + console.log(`Connector manifest:\n${JSON.stringify(readParams.manifest)}`); + console.log(`Config:\n${JSON.stringify(readParams.config)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + slices: [ + { + sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" }, + pages: [ + { + airbyteMessages: [ + { + type: "RECORD", + record: { + stream: readParams.stream, + data: { + id: "dp_123", + object: readParams.stream, + amount: 2000, + balance_transaction: "txn_123", + }, + }, + }, + { + type: "STATE", + state: { + data: { + timestamp: "2022-10-20T02:00:59Z", + }, + }, + }, + ], + request: { + url: "https://api.com/path", + }, + response: { + status: 200, + }, + }, + { + airbyteMessages: [ + { + type: "RECORD", + record: { + stream: readParams.stream, + data: { + id: "dp_123", + object: readParams.stream, + amount: 2000, + balance_transaction: "txn_123", + }, + }, + }, + { + type: "STATE", + state: { + data: { + timestamp: "2022-10-20T02:00:59Z", + }, + }, + }, + ], + request: { + url: "https://api.com/path", + }, + response: { + status: 200, + }, + }, + ], + }, + ], + }; + }); + } + + public listStreams(listParams: StreamsListRequestBody) { + // TODO: uncomment this once there is a real API to call + // return listStreams(listParams, this.requestOptions); + console.log(`Received listStreams body: ${JSON.stringify(listParams)}`); + return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { + return { + streams: [ + { + name: "disputes", + url: "https://api.com/disputes", + }, + { + name: "transactions", + url: "https://api.com/transactions", + }, + { + name: "users", + url: "https://api.com/users", + }, + ], + }; + }); + } +} diff --git a/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts b/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts deleted file mode 100644 index 69f0f9ca3f3d..000000000000 --- a/airbyte-webapp/src/services/connector-builder/ConnectorBuilderApiService.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { useQuery } from "react-query"; - -import { useConfig } from "config"; -import { AirbyteRequestService } from "core/request/AirbyteRequestService"; -import { - StreamRead, - StreamReadRequestBody, - StreamReadRequestBodyConfig, - StreamReadRequestBodyManifest, - StreamsListRead, - StreamsListRequestBody, -} from "core/request/ConnectorBuilderClient"; -import { useSuspenseQuery } from "services/connector/useSuspenseQuery"; -import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; -import { useInitService } from "services/useInitService"; - -class ConnectorBuilderService extends AirbyteRequestService { - public readStream(body: StreamReadRequestBody): Promise { - // TODO: uncomment this once there is a real API to call - // return readStream(body, this.requestOptions); - console.log("------------"); - console.log(`Stream: ${body.stream}`); - console.log(`Connector manifest:\n${JSON.stringify(body.manifest)}`); - console.log(`Config:\n${JSON.stringify(body.config)}`); - return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { - return { - slices: [ - { - sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" }, - pages: [ - { - airbyteMessages: [ - { - type: "RECORD", - record: { - stream: body.stream, - data: { - id: "dp_123", - object: body.stream, - amount: 2000, - balance_transaction: "txn_123", - }, - }, - }, - { - type: "STATE", - state: { - data: { - timestamp: "2022-10-20T02:00:59Z", - }, - }, - }, - ], - request: { - url: "https://api.com/path", - }, - response: { - status: 200, - }, - }, - { - airbyteMessages: [ - { - type: "RECORD", - record: { - stream: body.stream, - data: { - id: "dp_123", - object: body.stream, - amount: 2000, - balance_transaction: "txn_123", - }, - }, - }, - { - type: "STATE", - state: { - data: { - timestamp: "2022-10-20T02:00:59Z", - }, - }, - }, - ], - request: { - url: "https://api.com/path", - }, - response: { - status: 200, - }, - }, - ], - }, - ], - }; - }); - } - - public listStreams(body: StreamsListRequestBody): Promise { - // TODO: uncomment this once there is a real API to call - // return listStreams(body, this.requestOptions); - console.log(`Received listStreams body: ${JSON.stringify(body)}`); - return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { - return { - streams: [ - { - name: "disputes", - url: "https://api.com/disputes", - }, - { - name: "transactions", - url: "https://api.com/transactions", - }, - { - name: "users", - url: "https://api.com/users", - }, - ], - }; - }); - } -} - -const connectorBuilderKeys = { - all: ["connectorBuilder"] as const, - read: ( - streamName: string, - manifest: StreamReadRequestBodyManifest, - config: StreamReadRequestBodyConfig - ) => [...connectorBuilderKeys.all, "read", { streamName, manifest, config }] as const, - list: (manifest: StreamReadRequestBodyManifest) => - [...connectorBuilderKeys.all, "list", { manifest }] as const, -}; - -function useConnectorBuilderService() { - const config = useConfig(); - const middlewares = useDefaultRequestMiddlewares(); - return useInitService( - () => new ConnectorBuilderService(config.connectorBuilderApiUrl, middlewares), - [config.connectorBuilderApiUrl, middlewares] - ); -} - -export const useReadStream = (params: StreamReadRequestBody) => { - const service = useConnectorBuilderService(); - - return useQuery( - connectorBuilderKeys.read(params.stream, params.manifest, params.config), - () => service.readStream(params), - { refetchOnWindowFocus: false, enabled: false } - ); -}; - -export const useListStreams = (params: StreamsListRequestBody) => { - const service = useConnectorBuilderService(); - - return useSuspenseQuery(connectorBuilderKeys.list(params.manifest), () => service.listStreams(params)); -}; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts new file mode 100644 index 000000000000..c1fb27b0e745 --- /dev/null +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts @@ -0,0 +1,45 @@ +import { useQuery } from "react-query"; + +import { useConfig } from "config"; +import { ConnectorBuilderRequestService } from "core/domain/connectorBuilder/ConnectorBuilderRequestService"; +import { + StreamReadRequestBody, + StreamReadRequestBodyConfig, + StreamReadRequestBodyManifest, + StreamsListRequestBody, +} from "core/request/ConnectorBuilderClient"; +import { useSuspenseQuery } from "services/connector/useSuspenseQuery"; +import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; +import { useInitService } from "services/useInitService"; + +const connectorBuilderKeys = { + all: ["connectorBuilder"] as const, + read: (streamName: string, manifest: StreamReadRequestBodyManifest, config: StreamReadRequestBodyConfig) => + [...connectorBuilderKeys.all, "read", { streamName, manifest, config }] as const, + list: (manifest: StreamReadRequestBodyManifest) => [...connectorBuilderKeys.all, "list", { manifest }] as const, +}; + +function useConnectorBuilderService() { + const config = useConfig(); + const middlewares = useDefaultRequestMiddlewares(); + return useInitService( + () => new ConnectorBuilderRequestService(config.connectorBuilderApiUrl, middlewares), + [config.connectorBuilderApiUrl, middlewares] + ); +} + +export const useReadStream = (params: StreamReadRequestBody) => { + const service = useConnectorBuilderService(); + + return useQuery( + connectorBuilderKeys.read(params.stream, params.manifest, params.config), + () => service.readStream(params), + { refetchOnWindowFocus: false, enabled: false } + ); +}; + +export const useListStreams = (params: StreamsListRequestBody) => { + const service = useConnectorBuilderService(); + + return useSuspenseQuery(connectorBuilderKeys.list(params.manifest), () => service.listStreams(params)); +}; From 9d6a8fe92708c8f73002371a13dbb0f9ee485e4d Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 21 Oct 2022 13:45:31 -0700 Subject: [PATCH 6/9] fix name of pages folder --- .../ConnectorBuilderPage.tsx | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename airbyte-webapp/src/pages/{connector-builder => ConnectorBuilderPage}/ConnectorBuilderPage.tsx (100%) diff --git a/airbyte-webapp/src/pages/connector-builder/ConnectorBuilderPage.tsx b/airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx similarity index 100% rename from airbyte-webapp/src/pages/connector-builder/ConnectorBuilderPage.tsx rename to airbyte-webapp/src/pages/ConnectorBuilderPage/ConnectorBuilderPage.tsx From 4ae298ac75ec9bc2eb325c83ae292be8639e01b7 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 21 Oct 2022 13:46:09 -0700 Subject: [PATCH 7/9] improve comment --- .../domain/connectorBuilder/ConnectorBuilderRequestService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts index 18a997e31986..3c686d5b9677 100644 --- a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts +++ b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts @@ -4,7 +4,7 @@ import { AirbyteRequestService } from "../../request/AirbyteRequestService"; export class ConnectorBuilderRequestService extends AirbyteRequestService { public readStream(readParams: StreamReadRequestBody) { - // TODO: uncomment this once there is a real API to call + // TODO: uncomment this and remove mock responses once there is a real API to call // return readStream(readParams, this.requestOptions); console.log("------------"); console.log(`Stream: ${readParams.stream}`); @@ -84,7 +84,7 @@ export class ConnectorBuilderRequestService extends AirbyteRequestService { } public listStreams(listParams: StreamsListRequestBody) { - // TODO: uncomment this once there is a real API to call + // TODO: uncomment this and remove mock responses once there is a real API to call // return listStreams(listParams, this.requestOptions); console.log(`Received listStreams body: ${JSON.stringify(listParams)}`); return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { From 2599edac24f699c11040ca0407b9fca41414380f Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 21 Oct 2022 14:17:35 -0700 Subject: [PATCH 8/9] modify structure of StreamRead --- .../ConnectorBuilderRequestService.ts | 75 ++++++------------- .../src/main/openapi/openapi.yaml | 30 +++++--- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts index 3c686d5b9677..283df6e8f1c1 100644 --- a/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts +++ b/airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts @@ -1,9 +1,14 @@ -import { StreamReadRequestBody, StreamsListRequestBody } from "core/request/ConnectorBuilderClient"; +import { + StreamRead, + StreamReadRequestBody, + StreamsListRead, + StreamsListRequestBody, +} from "core/request/ConnectorBuilderClient"; import { AirbyteRequestService } from "../../request/AirbyteRequestService"; export class ConnectorBuilderRequestService extends AirbyteRequestService { - public readStream(readParams: StreamReadRequestBody) { + public readStream(readParams: StreamReadRequestBody): Promise { // TODO: uncomment this and remove mock responses once there is a real API to call // return readStream(readParams, this.requestOptions); console.log("------------"); @@ -12,60 +17,28 @@ export class ConnectorBuilderRequestService extends AirbyteRequestService { console.log(`Config:\n${JSON.stringify(readParams.config)}`); return new Promise((resolve) => setTimeout(resolve, 200)).then(() => { return { + logs: [ + { level: "INFO", message: "Syncing stream: rates " }, + { level: "INFO", message: "Setting state of rates stream to {'date': '2022-09-25'}" }, + ], slices: [ { sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" }, + state: { + type: "STREAM", + stream: { stream_descriptor: { name: readParams.stream }, stream_state: { date: "2022-09-26" } }, + data: { rates: { date: "2022-09-26" } }, + }, pages: [ { - airbyteMessages: [ - { - type: "RECORD", - record: { - stream: readParams.stream, - data: { - id: "dp_123", - object: readParams.stream, - amount: 2000, - balance_transaction: "txn_123", - }, - }, - }, - { - type: "STATE", - state: { - data: { - timestamp: "2022-10-20T02:00:59Z", - }, - }, - }, - ], - request: { - url: "https://api.com/path", - }, - response: { - status: 200, - }, - }, - { - airbyteMessages: [ - { - type: "RECORD", - record: { - stream: readParams.stream, - data: { - id: "dp_123", - object: readParams.stream, - amount: 2000, - balance_transaction: "txn_123", - }, - }, - }, + records: [ { - type: "STATE", - state: { - data: { - timestamp: "2022-10-20T02:00:59Z", - }, + stream: readParams.stream, + data: { + id: "dp_123", + object: readParams.stream, + amount: 2000, + balance_transaction: "txn_123", }, }, ], @@ -83,7 +56,7 @@ export class ConnectorBuilderRequestService extends AirbyteRequestService { }); } - public listStreams(listParams: StreamsListRequestBody) { + public listStreams(listParams: StreamsListRequestBody): Promise { // TODO: uncomment this and remove mock responses once there is a real API to call // return listStreams(listParams, this.requestOptions); console.log(`Received listStreams body: ${JSON.stringify(listParams)}`); diff --git a/connector-builder-server/src/main/openapi/openapi.yaml b/connector-builder-server/src/main/openapi/openapi.yaml index 19ce1ddd4bfd..9a6145e83989 100644 --- a/connector-builder-server/src/main/openapi/openapi.yaml +++ b/connector-builder-server/src/main/openapi/openapi.yaml @@ -63,40 +63,50 @@ components: StreamRead: type: object required: + - logs - slices properties: + logs: + type: array + description: The LOG AirbyteMessages that were emitted during the read of this slice + items: + type: object + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteLogMessage" slices: type: array - description: The stream slices returned from the read command + description: The stream slices returned from the read command. If no stream slicer is configured, this should contain a single item containing all of the results. items: type: object required: - - sliceDescriptor - pages properties: - sliceDescriptor: - type: object - description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}' pages: type: array - description: The pages returned from the read command + description: The pages returned from the read command. If no pagination is configured, this should contain a single item containing all of the results. items: type: object required: - - airbyteMessages + - records - request - response properties: - airbyteMessages: + records: type: array - description: The RECORD/STATE/LOG AirbyteMessages coming from the read operation for this page + description: The RECORD AirbyteMessages coming from the read operation for this page items: type: object - # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteMessage" + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteRecordMessage" request: $ref: "#/components/schemas/HttpRequest" response: $ref: "#/components/schemas/HttpResponse" + sliceDescriptor: + type: object + description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}. This can be omitted if a stream slicer is not configured.' + state: + type: object + description: The STATE AirbyteMessage emitted at the end of this slice. This can be omitted if a stream slicer is not configured. + # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage" StreamReadRequestBody: type: object required: From 6ab3272bb05c2109f49470078b3295d4f0a8c691 Mon Sep 17 00:00:00 2001 From: lmossman Date: Fri, 21 Oct 2022 14:24:41 -0700 Subject: [PATCH 9/9] fix path --- airbyte-webapp/src/pages/routes.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 1010703eba2c..ba76c4c73df1 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -15,7 +15,7 @@ import MainView from "views/layout/MainView"; import { WorkspaceRead } from "../core/request/AirbyteClient"; import ConnectionPage from "./ConnectionPage"; -import { ConnectorBuilderPage } from "./connector-builder/ConnectorBuilderPage"; +import { ConnectorBuilderPage } from "./ConnectorBuilderPage/ConnectorBuilderPage"; import DestinationPage from "./DestinationPage"; import OnboardingPage from "./OnboardingPage"; import PreferencesPage from "./PreferencesPage";