Generate connector builder api client

airbyte-webapp/.gitignore

@@ -29,5 +29,6 @@ yarn-error.log*
 
 storybook-static/
 
-# Ignore generated API client, since it's automatically generated
+# Ignore generated API clients, since they're automatically generated
 /src/core/request/AirbyteClient.ts
+/src/core/request/ConnectorBuilderClient.ts

airbyte-webapp/orval.config.ts

@@ -22,4 +22,25 @@ export default defineConfig({
       },
     },
   },
+  connectorBuilder: {
+    input: "../connector-builder-server/src/main/openapi/openapi.yaml",
+    output: {
+      target: "./src/core/request/ConnectorBuilderClient.ts",
+      prettier: true,
+      override: {
+        header: (info) => [
+          `eslint-disable`,
+          `Generated by orval 🍺`,
+          `Do not edit manually. Run "npm run generate-client" instead.`,
+          ...(info.title ? [info.title] : []),
+          ...(info.description ? [info.description] : []),
+          ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []),
+        ],
+        mutator: {
+          path: "./src/core/request/apiOverride.ts",
+          name: "apiOverride",
+        },
+      },
+    },
+  },
 });

airbyte-webapp/src/config/defaultConfig.ts

@@ -5,6 +5,7 @@ const defaultConfig: Config = {
   healthCheckInterval: 20000,
   version: "dev",
   apiUrl: `${window.location.protocol}//${window.location.hostname}:8001/api`,
+  connectorBuilderApiUrl: `${window.location.protocol}//${window.location.hostname}:8080/`,
   integrationUrl: "/docs",
   oauthRedirectUrl: `${window.location.protocol}//${window.location.host}`,
 };

airbyte-webapp/src/config/types.ts

@@ -21,6 +21,7 @@ declare global {
 export interface Config {
   segment: { token: string; enabled: boolean };
   apiUrl: string;
+  connectorBuilderApiUrl: string;
   oauthRedirectUrl: string;
   healthCheckInterval: number;
   version?: string;

airbyte-webapp/src/core/domain/connectorBuilder/ConnectorBuilderRequestService.ts

@@ -0,0 +1,82 @@
+import {
+  StreamRead,
+  StreamReadRequestBody,
+  StreamsListRead,
+  StreamsListRequestBody,
+} from "core/request/ConnectorBuilderClient";
+
+import { AirbyteRequestService } from "../../request/AirbyteRequestService";
+
+export class ConnectorBuilderRequestService extends AirbyteRequestService {
+  public readStream(readParams: StreamReadRequestBody): Promise<StreamRead> {
+    // TODO: uncomment this and remove mock responses once there is a real API to call
+    // return readStream(readParams, this.requestOptions);
+    console.log("------------");
+    console.log(`Stream: ${readParams.stream}`);
+    console.log(`Connector manifest:\n${JSON.stringify(readParams.manifest)}`);
+    console.log(`Config:\n${JSON.stringify(readParams.config)}`);
+    return new Promise((resolve) => setTimeout(resolve, 200)).then(() => {
+      return {
+        logs: [
+          { level: "INFO", message: "Syncing stream: rates " },
+          { level: "INFO", message: "Setting state of rates stream to {'date': '2022-09-25'}" },
+        ],
+        slices: [
+          {
+            sliceDescriptor: { start: "Jan 1, 2022", end: "Jan 2, 2022" },
+            state: {
+              type: "STREAM",
+              stream: { stream_descriptor: { name: readParams.stream }, stream_state: { date: "2022-09-26" } },
+              data: { rates: { date: "2022-09-26" } },
+            },
+            pages: [
+              {
+                records: [
+                  {
+                    stream: readParams.stream,
+                    data: {
+                      id: "dp_123",
+                      object: readParams.stream,
+                      amount: 2000,
+                      balance_transaction: "txn_123",
+                    },
+                  },
+                ],
+                request: {
+                  url: "https://api.com/path",
+                },
+                response: {
+                  status: 200,
+                },
+              },
+            ],
+          },
+        ],
+      };
+    });
+  }
+
+  public listStreams(listParams: StreamsListRequestBody): Promise<StreamsListRead> {
+    // TODO: uncomment this and remove mock responses once there is a real API to call
+    // return listStreams(listParams, this.requestOptions);
+    console.log(`Received listStreams body: ${JSON.stringify(listParams)}`);
+    return new Promise((resolve) => setTimeout(resolve, 200)).then(() => {
+      return {
+        streams: [
+          {
+            name: "disputes",
+            url: "https://api.com/disputes",
+          },
+          {
+            name: "transactions",
+            url: "https://api.com/transactions",
+          },
+          {
+            name: "users",
+            url: "https://api.com/users",
+          },
+        ],
+      };
+    });
+  }
+}

airbyte-webapp/src/pages/routes.tsx

@@ -15,7 +15,7 @@ import MainView from "views/layout/MainView";
 
 import { WorkspaceRead } from "../core/request/AirbyteClient";
 import ConnectionPage from "./ConnectionPage";
-import { ConnectorBuilderPage } from "./connector-builder/ConnectorBuilderPage";
+import { ConnectorBuilderPage } from "./ConnectorBuilderPage/ConnectorBuilderPage";
 import DestinationPage from "./DestinationPage";
 import OnboardingPage from "./OnboardingPage";
 import PreferencesPage from "./PreferencesPage";

airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderApiService.ts

@@ -0,0 +1,45 @@
+import { useQuery } from "react-query";
+
+import { useConfig } from "config";
+import { ConnectorBuilderRequestService } from "core/domain/connectorBuilder/ConnectorBuilderRequestService";
+import {
+  StreamReadRequestBody,
+  StreamReadRequestBodyConfig,
+  StreamReadRequestBodyManifest,
+  StreamsListRequestBody,
+} from "core/request/ConnectorBuilderClient";
+import { useSuspenseQuery } from "services/connector/useSuspenseQuery";
+import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares";
+import { useInitService } from "services/useInitService";
+
+const connectorBuilderKeys = {
+  all: ["connectorBuilder"] as const,
+  read: (streamName: string, manifest: StreamReadRequestBodyManifest, config: StreamReadRequestBodyConfig) =>
+    [...connectorBuilderKeys.all, "read", { streamName, manifest, config }] as const,
+  list: (manifest: StreamReadRequestBodyManifest) => [...connectorBuilderKeys.all, "list", { manifest }] as const,
+};
+
+function useConnectorBuilderService() {
+  const config = useConfig();
+  const middlewares = useDefaultRequestMiddlewares();
+  return useInitService(
+    () => new ConnectorBuilderRequestService(config.connectorBuilderApiUrl, middlewares),
+    [config.connectorBuilderApiUrl, middlewares]
+  );
+}
+
+export const useReadStream = (params: StreamReadRequestBody) => {
+  const service = useConnectorBuilderService();
+
+  return useQuery(
+    connectorBuilderKeys.read(params.stream, params.manifest, params.config),
+    () => service.readStream(params),
+    { refetchOnWindowFocus: false, enabled: false }
+  );
+};
+
+export const useListStreams = (params: StreamsListRequestBody) => {
+  const service = useConnectorBuilderService();
+
+  return useSuspenseQuery(connectorBuilderKeys.list(params.manifest), () => service.listStreams(params));
+};

connector-builder-server/src/main/openapi/openapi.yaml

@@ -18,6 +18,7 @@ paths:
   /v1/stream/read:
     post:
       summary: Reads a specific stream in the source. TODO in a later phase - only read a single slice of data.
+      operationId: readStream
       requestBody:
         content:
           application/json:
@@ -37,7 +38,8 @@ paths:
           $ref: "#/components/responses/InvalidInputResponse"
   /v1/streams/list:
     post:
-      summary: List all streams present in the connector definition, along with their specific request URLs
+      summary: List all streams present in the connector manifest, along with their specific request URLs
+      operationId: listStreams
       requestBody:
         content:
           application/json:
@@ -61,53 +63,71 @@ components:
     StreamRead:
       type: object
       required:
+        - logs
         - slices
       properties:
+        logs:
+          type: array
+          description: The LOG AirbyteMessages that were emitted during the read of this slice
+          items:
+            type: object
+            # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteLogMessage"
         slices:
           type: array
-          description: The stream slices returned from the read command
+          description: The stream slices returned from the read command. If no stream slicer is configured, this should contain a single item containing all of the results.
           items:
             type: object
             required:
-              - sliceDescriptor
               - pages
             properties:
-              sliceDescriptor:
-                type: object
-                description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}'
               pages:
                 type: array
-                description: The pages returned from the read command
+                description: The pages returned from the read command. If no pagination is configured, this should contain a single item containing all of the results.
                 items:
                   type: object
                   required:
-                    - airbyteMessages
+                    - records
                     - request
                     - response
                   properties:
-                    airbyteMessages:
+                    records:
                       type: array
-                      description: The RECORD/STATE/LOG AirbyteMessages coming from the read operation for this page
+                      description: The RECORD AirbyteMessages coming from the read operation for this page
                       items:
-                        $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteMessage"
+                        type: object
+                        # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteRecordMessage"
                     request:
                       $ref: "#/components/schemas/HttpRequest"
                     response:
                       $ref: "#/components/schemas/HttpResponse"
+              sliceDescriptor:
+                type: object
+                description: 'An object describing the current slice, e.g. {start_time: "2021-01-01", end_time: "2021-01-31"}. This can be omitted if a stream slicer is not configured.'
+              state:
+                type: object
+                description: The STATE AirbyteMessage emitted at the end of this slice. This can be omitted if a stream slicer is not configured.
+                # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage"
     StreamReadRequestBody:
       type: object
       required:
-        - definition
+        - manifest
         - stream
+        - config
       properties:
-        definition:
-          $ref: "#/components/schemas/ConnectorDefinitionBody"
-          description: The config-based connector definition contents
+        manifest:
+          type: object
+          description: The config-based connector manifest contents
+          # $ref: "#/components/schemas/ConnectorManifest"
         stream:
           type: string
           description: Name of the stream to read
+        config:
+          type: object
+          description: The config blob containing the user inputs for testing
         state:
-          $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage"
+          type: object
+          description: The AirbyteStateMessage object to use as the starting state for this read
+          # $ref: "#/components/schemas/AirbyteProtocol/definitions/AirbyteStateMessage"
         # --- Potential addition for a later phase ---
         # numPages:
         #   type: integer
@@ -144,18 +164,20 @@ components:
         headers:
           type: object
           description: The headers of the HTTP response, if any
-    ConnectorDefinitionBody:
-      $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json
-    AirbyteProtocol:
-      $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml
+    # --- Commenting out for now since they do not work with our orval openapi client generator ---
+    # ConnectorManifest:
+    #   $ref: ../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json
+    # AirbyteProtocol:
+    #   $ref: ../../../../airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml
     StreamsListRequestBody:
       type: object
       required:
-        - definition
+        - manifest
       properties:
-        definition:
-          $ref: "#/components/schemas/ConnectorDefinitionBody"
-          description: The config-based connector definition contents
+        manifest:
+          type: object
+          description: The config-based connector manifest contents
+          # $ref: "#/components/schemas/ConnectorManifest"
     StreamsListRead:
       type: object
       required:
@@ -165,7 +187,7 @@ components:
           type: array
           items:
             type: object
-            description: The stream names present in the connector definition
+            description: The stream names present in the connector manifest
             required:
               - name
               - url