Skip to content

Commit 0ddb12f

Browse files
geroplgeropl
committed
[server] Use workspace cluster as image-builder (feature flag: "movedImageBuilder")
1 parent 424410d commit 0ddb12f

File tree

7 files changed

+144
-18
lines changed

7 files changed

+144
-18
lines changed

‎components/image-builder-api/typescript/src/sugar.ts

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,13 @@ import {
2828
import { injectable, inject, optional } from "inversify";
2929
import * as grpc from "@grpc/grpc-js";
3030
import { TextDecoder } from "util";
31-
import { ImageBuildLogInfo } from "@gitpod/gitpod-protocol";
31+
import { ImageBuildLogInfo, User, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
3232

3333
export const ImageBuilderClientProvider = Symbol("ImageBuilderClientProvider");
3434

3535
// ImageBuilderClientProvider caches image builder connections
3636
export interface ImageBuilderClientProvider {
37-
getDefault(): PromisifiedImageBuilderClient;
37+
getClient(user: User, workspace: Workspace, instance?: WorkspaceInstance): Promise<PromisifiedImageBuilderClient>;
3838
}
3939

4040
function withTracing(ctx: TraceContext) {
@@ -93,6 +93,22 @@ export class CachingImageBuilderClientProvider implements ImageBuilderClientProv
9393
this.connectionCache = connection;
9494
return connection;
9595
}
96+
97+
async getClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
98+
return this.getDefault();
99+
}
100+
101+
promisify(c: ImageBuilderClient): PromisifiedImageBuilderClient {
102+
let interceptors: grpc.Interceptor[] = [];
103+
if (this.clientCallMetrics) {
104+
interceptors = [createClientCallMetricsInterceptor(this.clientCallMetrics)];
105+
}
106+
107+
return new PromisifiedImageBuilderClient(
108+
new ImageBuilderClient(this.clientConfig.address, grpc.credentials.createInsecure()),
109+
interceptors,
110+
);
111+
}
96112
}
97113

98114
// StagedBuildResponse captures the multi-stage nature (starting, running, done) of image builds.

‎components/server/src/container-module.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ import { ReferrerPrefixParser } from "./workspace/referrer-prefix-context-parser
9999
import { InstallationAdminTelemetryDataProvider } from "./installation-admin/telemetry-data-provider";
100100
import { IDEService } from "./ide-service";
101101
import { LicenseEvaluator } from "@gitpod/licensor/lib";
102+
import { WorkspaceClusterImagebuilderClientProvider } from "./workspace/workspace-cluster-imagebuilder-client-provider";
102103

103104
export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
104105
bind(Config).toConstantValue(ConfigFile.fromFile());
@@ -162,6 +163,7 @@ export const productionContainerModule = new ContainerModule((bind, unbind, isBo
162163
return { address: config.imageBuilderAddr };
163164
});
164165
bind(CachingImageBuilderClientProvider).toSelf().inSingletonScope();
166+
bind(WorkspaceClusterImagebuilderClientProvider).toSelf().inSingletonScope(); // during the transition period, we have two kinds of image builder client providers
165167
bind(ImageBuilderClientProvider).toService(CachingImageBuilderClientProvider);
166168
bind(ImageBuilderClientCallMetrics).toService(IClientCallMetrics);
167169

‎components/server/src/workspace/gitpod-server-impl.ts

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ import { LicenseEvaluator } from "@gitpod/licensor/lib";
162162
import { Feature } from "@gitpod/licensor/lib/api";
163163
import { Currency } from "@gitpod/gitpod-protocol/lib/plans";
164164
import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
165+
import { WorkspaceClusterImagebuilderClientProvider } from "./workspace-cluster-imagebuilder-client-provider";
165166

166167
// shortcut
167168
export const traceWI = (ctx: TraceContext, wi: Omit<LogContext, "userId">) => TraceContext.setOWI(ctx, wi); // userId is already taken care of in WebsocketConnectionManager
@@ -193,7 +194,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
193194
@inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter;
194195
@inject(WorkspaceManagerClientProvider)
195196
protected readonly workspaceManagerClientProvider: WorkspaceManagerClientProvider;
196-
@inject(ImageBuilderClientProvider) protected imageBuilderClientProvider: ImageBuilderClientProvider;
197+
@inject(ImageBuilderClientProvider) protected imagebuilderClientProvider: ImageBuilderClientProvider;
198+
@inject(WorkspaceClusterImagebuilderClientProvider)
199+
protected readonly wsClusterImageBuilderClientProvider: ImageBuilderClientProvider;
197200

198201
@inject(UserDB) protected readonly userDB: UserDB;
199202
@inject(TokenProvider) protected readonly tokenProvider: TokenProvider;
@@ -1569,7 +1572,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
15691572
// a change to move the imageBuildLogInfo across the globe.
15701573
log.warn(logCtx, "imageBuild logs: fallback!");
15711574
ctx.span?.setTag("workspace.imageBuild.logs.fallback", true);
1572-
await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, workspace);
1575+
await this.deprecatedDoWatchWorkspaceImageBuildLogs(ctx, logCtx, user, workspace);
15731576
return;
15741577
}
15751578

@@ -1619,6 +1622,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
16191622
protected async deprecatedDoWatchWorkspaceImageBuildLogs(
16201623
ctx: TraceContext,
16211624
logCtx: LogContext,
1625+
user: User,
16221626
workspace: Workspace,
16231627
) {
16241628
if (!workspace.imageNameResolved) {
@@ -1627,7 +1631,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
16271631
}
16281632

16291633
try {
1630-
const imgbuilder = this.imageBuilderClientProvider.getDefault();
1634+
const imgbuilder = await this.getImageBuilderClient(user, workspace, undefined);
16311635
const req = new LogsRequest();
16321636
req.setCensored(true);
16331637
req.setBuildRef(workspace.imageNameResolved);
@@ -3104,4 +3108,23 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
31043108
}
31053109
//
31063110
//#endregion
3111+
3112+
/**
3113+
* This method is temporary until we moved image-builder into workspace clusters
3114+
* @param user
3115+
* @param workspace
3116+
* @param instance
3117+
* @returns
3118+
*/
3119+
protected async getImageBuilderClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
3120+
const isMovedImageBuilder = await getExperimentsClientForBackend().getValueAsync("movedImageBuilder", false, {
3121+
userId: user.id,
3122+
projectId: workspace.projectId,
3123+
});
3124+
if (isMovedImageBuilder) {
3125+
return this.wsClusterImageBuilderClientProvider.getClient(user, workspace, instance);
3126+
} else {
3127+
return this.imagebuilderClientProvider.getClient(user, workspace, instance);
3128+
}
3129+
}
31073130
}

‎components/server/src/workspace/workspace-cluster-imagebuilder-client-provider.ts

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/**
2+
* Copyright (c) 2022 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License-AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
8+
import { IClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics";
9+
import { defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc";
10+
import {
11+
ImageBuilderClient,
12+
ImageBuilderClientCallMetrics,
13+
ImageBuilderClientProvider,
14+
PromisifiedImageBuilderClient,
15+
} from "@gitpod/image-builder/lib";
16+
import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
17+
import {
18+
WorkspaceManagerClientProviderCompositeSource,
19+
WorkspaceManagerClientProviderSource,
20+
} from "@gitpod/ws-manager/lib/client-provider-source";
21+
import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
22+
import { inject, injectable, optional } from "inversify";
23+
24+
@injectable()
25+
export class WorkspaceClusterImagebuilderClientProvider implements ImageBuilderClientProvider {
26+
@inject(WorkspaceManagerClientProviderCompositeSource)
27+
protected readonly source: WorkspaceManagerClientProviderSource;
28+
@inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider;
29+
@inject(ImageBuilderClientCallMetrics) @optional() protected readonly clientCallMetrics: IClientCallMetrics;
30+
31+
// gRPC connections can be used concurrently, even across services.
32+
// Thus it makes sense to cache them rather than create a new connection for each request.
33+
protected readonly connectionCache = new Map<string, ImageBuilderClient>();
34+
35+
async getClient(
36+
user: ExtendedUser,
37+
workspace: Workspace,
38+
instance: WorkspaceInstance,
39+
): Promise<PromisifiedImageBuilderClient> {
40+
const clusters = await this.clientProvider.getStartClusterSets(user, workspace, instance);
41+
for await (let cluster of clusters) {
42+
const info = await this.source.getWorkspaceCluster(cluster.installation);
43+
if (!info) {
44+
continue;
45+
}
46+
47+
var client = this.connectionCache.get(info.name);
48+
if (!client) {
49+
client = this.clientProvider.createConnection(ImageBuilderClient, info, defaultGRPCOptions);
50+
this.connectionCache.set(info.name, client);
51+
}
52+
return new PromisifiedImageBuilderClient(client, []);
53+
}
54+
55+
throw new Error("no image-builder available");
56+
}
57+
}

‎components/server/src/workspace/workspace-starter.ts

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
114114
import { increaseFailedInstanceStartCounter, increaseSuccessfulInstanceStartCounter } from "../prometheus-metrics";
115115
import { ContextParser } from "./context-parser-service";
116116
import { IDEService } from "../ide-service";
117+
import { WorkspaceClusterImagebuilderClientProvider } from "./workspace-cluster-imagebuilder-client-provider";
118+
import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
117119

118120
export interface StartWorkspaceOptions {
119121
rethrow?: boolean;
@@ -197,6 +199,8 @@ export class WorkspaceStarter {
197199
@inject(MessageBusIntegration) protected readonly messageBus: MessageBusIntegration;
198200
@inject(AuthorizationService) protected readonly authService: AuthorizationService;
199201
@inject(ImageBuilderClientProvider) protected readonly imagebuilderClientProvider: ImageBuilderClientProvider;
202+
@inject(WorkspaceClusterImagebuilderClientProvider)
203+
protected readonly wsClusterImageBuilderClientProvider: ImageBuilderClientProvider;
200204
@inject(ImageSourceProvider) protected readonly imageSourceProvider: ImageSourceProvider;
201205
@inject(UserService) protected readonly userService: UserService;
202206
@inject(IAnalyticsWriter) protected readonly analytics: IAnalyticsWriter;
@@ -256,7 +260,7 @@ export class WorkspaceStarter {
256260
auth.setTotal(allowAll);
257261
req.setAuth(auth);
258262

259-
const client = this.imagebuilderClientProvider.getDefault();
263+
const client = await this.getImageBuilderClient(user, workspace, undefined);
260264
const res = await client.resolveBaseImage({ span }, req);
261265
workspace.imageSource = <WorkspaceImageSourceReference>{
262266
baseImageResolved: res.getRef(),
@@ -974,7 +978,7 @@ export class WorkspaceStarter {
974978
): Promise<boolean> {
975979
const span = TraceContext.startSpan("needsImageBuild", ctx);
976980
try {
977-
const client = this.imagebuilderClientProvider.getDefault();
981+
const client = await this.getImageBuilderClient(user, workspace, instance);
978982
const { src, auth, disposable } = await this.prepareBuildRequest(
979983
{ span },
980984
workspace,
@@ -1014,7 +1018,7 @@ export class WorkspaceStarter {
10141018

10151019
try {
10161020
// Start build...
1017-
const client = this.imagebuilderClientProvider.getDefault();
1021+
const client = await this.getImageBuilderClient(user, workspace, instance);
10181022
const { src, auth, disposable } = await this.prepareBuildRequest(
10191023
{ span },
10201024
workspace,
@@ -1746,4 +1750,23 @@ export class WorkspaceStarter {
17461750

17471751
return result;
17481752
}
1753+
1754+
/**
1755+
* This method is temporary until we moved image-builder into workspace clusters
1756+
* @param user
1757+
* @param workspace
1758+
* @param instance
1759+
* @returns
1760+
*/
1761+
protected async getImageBuilderClient(user: User, workspace: Workspace, instance?: WorkspaceInstance) {
1762+
const isMovedImageBuilder = await getExperimentsClientForBackend().getValueAsync("movedImageBuilder", false, {
1763+
userId: user.id,
1764+
projectId: workspace.projectId,
1765+
});
1766+
if (isMovedImageBuilder) {
1767+
return this.wsClusterImageBuilderClientProvider.getClient(user, workspace, instance);
1768+
} else {
1769+
return this.imagebuilderClientProvider.getClient(user, workspace, instance);
1770+
}
1771+
}
17491772
}

‎components/ws-manager-api/typescript/src/client-provider.ts

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,14 @@ export class WorkspaceManagerClientProvider implements Disposable {
105105
let client = this.connectionCache.get(name);
106106
if (!client) {
107107
const info = await getConnectionInfo();
108-
client = this.createClient(info, grpcOptions);
108+
client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
109109
this.connectionCache.set(name, client);
110110
} else if (client.getChannel().getConnectivityState(true) != grpc.connectivityState.READY) {
111111
client.close();
112112

113113
console.warn(`Lost connection to workspace manager \"${name}\" - attempting to reestablish`);
114114
const info = await getConnectionInfo();
115-
client = this.createClient(info, grpcOptions);
115+
client = this.createConnection(WorkspaceManagerClient, info, grpcOptions);
116116
this.connectionCache.set(name, client);
117117
}
118118

@@ -137,7 +137,16 @@ export class WorkspaceManagerClientProvider implements Disposable {
137137
return this.source.getAllWorkspaceClusters();
138138
}
139139

140-
public createClient(info: WorkspaceManagerConnectionInfo, grpcOptions?: object): WorkspaceManagerClient {
140+
public createConnection<T extends grpc.Client>(
141+
creator: { new (address: string, credentials: grpc.ChannelCredentials, options?: grpc.ClientOptions): T },
142+
info: WorkspaceManagerConnectionInfo,
143+
grpcOptions?: object,
144+
): T {
145+
const options: Partial<grpc.ClientOptions> = {
146+
...grpcOptions,
147+
"grpc.ssl_target_name_override": "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager"
148+
};
149+
141150
let credentials: grpc.ChannelCredentials;
142151
if (info.tls) {
143152
const rootCerts = Buffer.from(info.tls.ca, "base64");
@@ -149,11 +158,7 @@ export class WorkspaceManagerClientProvider implements Disposable {
149158
credentials = grpc.credentials.createInsecure();
150159
}
151160

152-
const options: Partial<grpc.ClientOptions> = {
153-
...grpcOptions,
154-
"grpc.ssl_target_name_override": "ws-manager", // this makes sure we can call ws-manager with a URL different to "ws-manager"
155-
};
156-
return new WorkspaceManagerClient(info.url, credentials, options);
161+
return new creator(info.url, credentials, options);
157162
}
158163

159164
public dispose() {

‎components/ws-manager-bridge/src/cluster-service-server.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import {
3434
UpdateResponse,
3535
AdmissionConstraint as GRPCAdmissionConstraint,
3636
} from "@gitpod/ws-manager-bridge-api/lib";
37-
import { GetWorkspacesRequest } from "@gitpod/ws-manager/lib";
37+
import { GetWorkspacesRequest, WorkspaceManagerClient } from "@gitpod/ws-manager/lib";
3838
import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
3939
import {
4040
WorkspaceManagerClientProviderCompositeSource,
@@ -151,7 +151,7 @@ export class ClusterService implements IClusterServiceServer {
151151

152152
// try to connect to validate the config. Throws an exception if it fails.
153153
await new Promise<void>((resolve, reject) => {
154-
const c = this.clientProvider.createClient(newCluster);
154+
const c = this.clientProvider.createConnection(WorkspaceManagerClient, newCluster);
155155
c.getWorkspaces(new GetWorkspacesRequest(), (err: any) => {
156156
if (err) {
157157
reject(

0 commit comments

Comments
 (0)