[server] Use workspace cluster as image-builder (feature flag: "movedImageBuilder")

Author: geropl

components/image-builder-api/typescript/src/sugar.ts

@@ -28,13 +28,13 @@ import {
 import { injectable, inject, optional } from "inversify";
 import * as grpc from "@grpc/grpc-js";
 import { TextDecoder } from "util";
-import { ImageBuildLogInfo } from "@gitpod/gitpod-protocol";
+import { ImageBuildLogInfo, User, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
 
 export const ImageBuilderClientProvider = Symbol("ImageBuilderClientProvider");
 
 // ImageBuilderClientProvider caches image builder connections
 export interface ImageBuilderClientProvider {
-    getDefault(): PromisifiedImageBuilderClient;
+    getClient(user: User, workspace: Workspace, instance?: WorkspaceInstance): Promise<PromisifiedImageBuilderClient>;
 }
 
 function withTracing(ctx: TraceContext) {
@@ -93,6 +93,22 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv
         this.connectionCache = connection;
         return connection;
     }
+
+    async getClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
+        return this.getDefault();
+    }
+
+    promisify(c: ImageBuilderClient): PromisifiedImageBuilderClient {
+        let interceptors: grpc.Interceptor[] = [];
+        if (this.clientCallMetrics) {
+            interceptors = [createClientCallMetricsInterceptor(this.clientCallMetrics)];
+        }
+
+        return new PromisifiedImageBuilderClient(
+            new ImageBuilderClient(this.clientConfig.address, grpc.credentials.createInsecure()),
+            interceptors,
+        );
+    }
 }
 
 // StagedBuildResponse captures the multi-stage nature (starting, running, done) of image builds.

components/server/src/container-module.ts

@@ -99,6 +99,7 @@ import { ReferrerPrefixParser } from "./workspace/referrer-prefix-context-parser
 import { InstallationAdminTelemetryDataProvider } from "./installation-admin/telemetry-data-provider";
 import { IDEService } from "./ide-service";
 import { LicenseEvaluator } from "@gitpod/licensor/lib";
+import { WorkspaceClusterImagebuilderClientProvider } from "./workspace/workspace-cluster-imagebuilder-client-provider";
 
 export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(Config).toConstantValue(ConfigFile.fromFile());
@@ -162,6 +163,7 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo
         return { address: config.imageBuilderAddr };
     });
     bind(CachingImageBuilderClientProvider).toSelf().inSingletonScope();
+    bind(WorkspaceClusterImagebuilderClientProvider).toSelf().inSingletonScope(); // during the transition period, we have two kinds of image builder client providers
     bind(ImageBuilderClientProvider).toService(CachingImageBuilderClientProvider);
     bind(ImageBuilderClientCallMetrics).toService(IClientCallMetrics);
 

components/server/src/workspace/gitpod-server-impl.ts

@@ -162,6 +162,7 @@ import { LicenseEvaluator } from "@gitpod/licensor/lib";
 import { Feature } from "@gitpod/licensor/lib/api";
 import { Currency } from "@gitpod/gitpod-protocol/lib/plans";
 import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
+import { WorkspaceClusterImagebuilderClientProvider } from "./workspace-cluster-imagebuilder-client-provider";
 
 // shortcut
 export const traceWI = (ctx: TraceContext, wi: Omit<LogContext, "userId">) => TraceContext.setOWI(ctx, wi); // userId is already taken care of in WebsocketConnectionManager
@@ -193,7 +194,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
     @inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter;
     @inject(WorkspaceManagerClientProvider)
     protected readonly workspaceManagerClientProvider: WorkspaceManagerClientProvider;
-    @inject(ImageBuilderClientProvider) protected imageBuilderClientProvider: ImageBuilderClientProvider;
+    @inject(ImageBuilderClientProvider) protected imagebuilderClientProvider: ImageBuilderClientProvider;
+    @inject(WorkspaceClusterImagebuilderClientProvider)
+    protected readonly wsClusterImageBuilderClientProvider: ImageBuilderClientProvider;
 
     @inject(UserDB) protected readonly userDB: UserDB;
     @inject(TokenProvider) protected readonly tokenProvider: TokenProvider;
@@ -1569,7 +1572,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
             // a change to move the imageBuildLogInfo across the globe.
             log.warn(logCtx, "imageBuild logs: fallback!");
             ctx.span?.setTag("workspace.imageBuild.logs.fallback", true);
-            await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, workspace);
+            await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, user, workspace);
             return;
         }
 
@@ -1619,6 +1622,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
     protected async deprecatedDoWatchWorkspaceImageBuildLogs(
         ctx: TraceContext,
         logCtx: LogContext,
+        user: User,
         workspace: Workspace,
     ) {
         if (!workspace.imageNameResolved) {
@@ -1627,7 +1631,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
         }
 
         try {
-            const imgbuilder = this.imageBuilderClientProvider.getDefault();
+            const imgbuilder = await this.getImageBuilderClient(user, workspace, undefined);
             const req = new LogsRequest();
             req.setCensored(true);
             req.setBuildRef(workspace.imageNameResolved);
@@ -3104,4 +3108,23 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
     }
     //
     //#endregion
+
+    /**
+     * This method is temporary until we moved image-builder into workspace clusters
+     * @param user
+     * @param workspace
+     * @param instance
+     * @returns
+     */
+    protected async getImageBuilderClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
+        const isMovedImageBuilder = await getExperimentsClientForBackend().getValueAsync("movedImageBuilder", false, {
+            userId: user.id,
+            projectId: workspace.projectId,
+        });
+        if (isMovedImageBuilder) {
+            return this.wsClusterImageBuilderClientProvider.getClient(user, workspace, instance);
+        } else {
+            return this.imagebuilderClientProvider.getClient(user, workspace, instance);
+        }
+    }
 }

components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
+import { IClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics";
+import { defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc";
+import {
+    ImageBuilderClient,
+    ImageBuilderClientCallMetrics,
+    ImageBuilderClientProvider,
+    PromisifiedImageBuilderClient,
+} from "@gitpod/image-builder/lib";
+import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
+import {
+    WorkspaceManagerClientProviderCompositeSource,
+    WorkspaceManagerClientProviderSource,
+} from "@gitpod/ws-manager/lib/client-provider-source";
+import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
+import { inject, injectable, optional } from "inversify";
+
+@injectable()
+export class WorkspaceClusterImagebuilderClientProvider implements ImageBuilderClientProvider {
+    @inject(WorkspaceManagerClientProviderCompositeSource)
+    protected readonly source: WorkspaceManagerClientProviderSource;
+    @inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider;
+    @inject(ImageBuilderClientCallMetrics) @optional() protected readonly clientCallMetrics: IClientCallMetrics;
+
+    // gRPC connections can be used concurrently, even across services.
+    // Thus it makes sense to cache them rather than create a new connection for each request.
+    protected readonly connectionCache = new Map<string, ImageBuilderClient>();
+
+    async getClient(
+        user: ExtendedUser,
+        workspace: Workspace,
+        instance: WorkspaceInstance,
+    ): Promise<PromisifiedImageBuilderClient> {
+        const clusters = await this.clientProvider.getStartClusterSets(user, workspace, instance);
+        for await (let cluster of clusters) {
+            const info = await this.source.getWorkspaceCluster(cluster.installation);
+            if (!info) {
+                continue;
+            }
+
+            var client = this.connectionCache.get(info.name);
+            if (!client) {
+                client = this.clientProvider.createConnection(ImageBuilderClient, info, defaultGRPCOptions);
+                this.connectionCache.set(info.name, client);
+            }
+            return new PromisifiedImageBuilderClient(client, []);
+        }
+
+        throw new Error("no image-builder available");
+    }
+}

components/server/src/workspace/workspace-starter.ts

@@ -114,6 +114,8 @@ import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
 import { increaseFailedInstanceStartCounter, increaseSuccessfulInstanceStartCounter } from "../prometheus-metrics";
 import { ContextParser } from "./context-parser-service";
 import { IDEService } from "../ide-service";
+import { WorkspaceClusterImagebuilderClientProvider } from "./workspace-cluster-imagebuilder-client-provider";
+import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
 
 export interface StartWorkspaceOptions {
     rethrow?: boolean;
@@ -197,6 +199,8 @@ export class WorkspaceStarter {
     @inject(MessageBusIntegration) protected readonly messageBus: MessageBusIntegration;
     @inject(AuthorizationService) protected readonly authService: AuthorizationService;
     @inject(ImageBuilderClientProvider) protected readonly imagebuilderClientProvider: ImageBuilderClientProvider;
+    @inject(WorkspaceClusterImagebuilderClientProvider)
+    protected readonly wsClusterImageBuilderClientProvider: ImageBuilderClientProvider;
     @inject(ImageSourceProvider) protected readonly imageSourceProvider: ImageSourceProvider;
     @inject(UserService) protected readonly userService: UserService;
     @inject(IAnalyticsWriter) protected readonly analytics: IAnalyticsWriter;
@@ -256,7 +260,7 @@ export class WorkspaceStarter {
                 auth.setTotal(allowAll);
                 req.setAuth(auth);
 
-                const client = this.imagebuilderClientProvider.getDefault();
+                const client = await this.getImageBuilderClient(user, workspace, undefined);
                 const res = await client.resolveBaseImage({ span }, req);
                 workspace.imageSource = <WorkspaceImageSourceReference>{
                     baseImageResolved: res.getRef(),
@@ -974,7 +978,7 @@ export class WorkspaceStarter {
     ): Promise<boolean> {
         const span = TraceContext.startSpan("needsImageBuild", ctx);
         try {
-            const client = this.imagebuilderClientProvider.getDefault();
+            const client = await this.getImageBuilderClient(user, workspace, instance);
             const { src, auth, disposable } = await this.prepareBuildRequest(
                 { span },
                 workspace,
@@ -1014,7 +1018,7 @@ export class WorkspaceStarter {
 
         try {
             // Start build...
-            const client = this.imagebuilderClientProvider.getDefault();
+            const client = await this.getImageBuilderClient(user, workspace, instance);
             const { src, auth, disposable } = await this.prepareBuildRequest(
                 { span },
                 workspace,
@@ -1746,4 +1750,23 @@ export class WorkspaceStarter {
 
         return result;
     }
+
+    /**
+     * This method is temporary until we moved image-builder into workspace clusters
+     * @param user
+     * @param workspace
+     * @param instance
+     * @returns
+     */
+    protected async getImageBuilderClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
+        const isMovedImageBuilder = await getExperimentsClientForBackend().getValueAsync("movedImageBuilder", false, {
+            userId: user.id,
+            projectId: workspace.projectId,
+        });
+        if (isMovedImageBuilder) {
+            return this.wsClusterImageBuilderClientProvider.getClient(user, workspace, instance);
+        } else {
+            return this.imagebuilderClientProvider.getClient(user, workspace, instance);
+        }
+    }
 }

components/ws-manager-api/typescript/src/client-provider.ts

@@ -105,14 +105,14 @@ export class WorkspaceManagerClientProvider implements Disposable {
         let client = this.connectionCache.get(name);
         if (!client) {
             const info = await getConnectionInfo();
-            client = this.createClient(info, grpcOptions);
+            client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
             this.connectionCache.set(name, client);
         } else if (client.getChannel().getConnectivityState(true) != grpc.connectivityState.READY) {
             client.close();
 
             console.warn(`Lost connection to workspace manager \"${name}\" - attempting to reestablish`);
             const info = await getConnectionInfo();
-            client = this.createClient(info, grpcOptions);
+            client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
             this.connectionCache.set(name, client);
         }
 
@@ -137,7 +137,16 @@ export class WorkspaceManagerClientProvider implements Disposable {
         return this.source.getAllWorkspaceClusters();
     }
 
-    public createClient(info: WorkspaceManagerConnectionInfo, grpcOptions?: object): WorkspaceManagerClient {
+    public createConnection<T extends grpc.Client>(
+        creator: { new (address: string, credentials: grpc.ChannelCredentials, options?: grpc.ClientOptions): T },
+        info: WorkspaceManagerConnectionInfo,
+        grpcOptions?: object,
+    ): T {
+        const options: Partial<grpc.ClientOptions> = {
+            ...grpcOptions,
+            "grpc.ssl_target_name_override": "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager"
+        };
+
         let credentials: grpc.ChannelCredentials;
         if (info.tls) {
             const rootCerts = Buffer.from(info.tls.ca, "base64");
@@ -149,11 +158,7 @@ export class WorkspaceManagerClientProvider implements Disposable {
             credentials = grpc.credentials.createInsecure();
         }
 
-        const options: Partial<grpc.ClientOptions> = {
-            ...grpcOptions,
-            "grpc.ssl_target_name_override": "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager"
-        };
-        return new WorkspaceManagerClient(info.url, credentials, options);
+        return new creator(info.url, credentials, options);
     }
 
     public dispose() {

components/ws-manager-bridge/src/cluster-service-server.ts

@@ -34,7 +34,7 @@ import {
     UpdateResponse,
     AdmissionConstraint as GRPCAdmissionConstraint,
 } from "@gitpod/ws-manager-bridge-api/lib";
-import { GetWorkspacesRequest } from "@gitpod/ws-manager/lib";
+import { GetWorkspacesRequest, WorkspaceManagerClient } from "@gitpod/ws-manager/lib";
 import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
 import {
     WorkspaceManagerClientProviderCompositeSource,
@@ -151,7 +151,7 @@ export class ClusterService implements IClusterServiceServer {
 
                 // try to connect to validate the config. Throws an exception if it fails.
                 await new Promise<void>((resolve, reject) => {
-                    const c = this.clientProvider.createClient(newCluster);
+                    const c = this.clientProvider.createConnection(WorkspaceManagerClient, newCluster);
                     c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => {
                         if (err) {
                             reject(