From 988649144df902e29a66c46d72dac97626074f84 Mon Sep 17 00:00:00 2001 From: Jan Keromnes Date: Wed, 19 Oct 2022 13:35:22 +0200 Subject: [PATCH 1/3] Revert "[server] Don't fetch the repository config a second time when starting incremental prebuilds" This reverts commit 53ad76132823e874479c8226d666929e85b0e36a. --- .../server/ee/src/prebuilds/incremental-prebuilds-service.ts | 2 +- components/server/ee/src/workspace/gitpod-server-impl.ts | 4 ---- components/server/ee/src/workspace/workspace-factory.ts | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts b/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts index 4f176079f4358e..84fbeffdefb7e5 100644 --- a/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts +++ b/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts @@ -66,7 +66,6 @@ export class IncrementalPrebuildsService { public async findGoodBaseForIncrementalBuild( context: CommitContext, - config: WorkspaceConfig, history: WithCommitHistory, user: User, ): Promise { @@ -74,6 +73,7 @@ export class IncrementalPrebuildsService { return; } + const { config } = await this.configProvider.fetchConfig({}, user, context); const imageSource = await this.imageSourceProvider.getImageSource({}, user, context, config); // Note: This query returns only not-garbage-collected prebuilds in order to reduce cardinality diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index fb25aeecccdf0d..eb8526110d8418 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -119,13 +119,11 @@ import { UsageServiceDefinition } from "@gitpod/usage-api/lib/usage/v1/usage.pb" import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server"; import { BillingServiceClient, BillingServiceDefinition } from "@gitpod/usage-api/lib/usage/v1/billing.pb"; import { IncrementalPrebuildsService } from "../prebuilds/incremental-prebuilds-service"; -import { ConfigProvider } from "../../../src/workspace/config-provider"; @injectable() export class GitpodServerEEImpl extends GitpodServerImpl { @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; @inject(IncrementalPrebuildsService) protected readonly incrementalPrebuildsService: IncrementalPrebuildsService; - @inject(ConfigProvider) protected readonly configProvider: ConfigProvider; @inject(LicenseDB) protected readonly licenseDB: LicenseDB; @inject(LicenseKeySource) protected readonly licenseKeySource: LicenseKeySource; @@ -989,11 +987,9 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace }; log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); if (prebuiltWorkspace?.state !== "available" && mode === CreateWorkspaceMode.UseLastSuccessfulPrebuild) { - const { config } = await this.configProvider.fetchConfig({}, user, context); const history = await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); prebuiltWorkspace = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( context, - config, history, user, ); diff --git a/components/server/ee/src/workspace/workspace-factory.ts b/components/server/ee/src/workspace/workspace-factory.ts index 35c86f6d98eeab..8596054fd351b4 100644 --- a/components/server/ee/src/workspace/workspace-factory.ts +++ b/components/server/ee/src/workspace/workspace-factory.ts @@ -121,7 +121,6 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { let ws; const recentPrebuild = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( commitContext, - config, context, user, ); From 268430364a61daa29c8360e16dfe6a880b262836 Mon Sep 17 00:00:00 2001 From: Jan Keromnes Date: Wed, 19 Oct 2022 11:37:04 +0000 Subject: [PATCH 2/3] Revert "[dashboard] Introduce showUseLastSuccessfulPrebuild feature flag" This reverts commit 1454544efc7eac542a4ad35ec68c8ec1553a2c13. --- .../src/contexts/FeatureFlagContext.tsx | 8 +------- .../dashboard/src/start/CreateWorkspace.tsx | 16 ++++++---------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/components/dashboard/src/contexts/FeatureFlagContext.tsx b/components/dashboard/src/contexts/FeatureFlagContext.tsx index f6dc1acd4fade2..df57ac67ba7bbf 100644 --- a/components/dashboard/src/contexts/FeatureFlagContext.tsx +++ b/components/dashboard/src/contexts/FeatureFlagContext.tsx @@ -18,11 +18,9 @@ interface FeatureFlagConfig { const FeatureFlagContext = createContext<{ showPersistentVolumeClaimUI: boolean; showUsageView: boolean; - showUseLastSuccessfulPrebuild: boolean; }>({ showPersistentVolumeClaimUI: false, showUsageView: false, - showUseLastSuccessfulPrebuild: false, }); const FeatureFlagContextProvider: React.FC = ({ children }) => { @@ -33,7 +31,6 @@ const FeatureFlagContextProvider: React.FC = ({ children }) => { const team = getCurrentTeam(location, teams); const [showPersistentVolumeClaimUI, setShowPersistentVolumeClaimUI] = useState(false); const [showUsageView, setShowUsageView] = useState(false); - const [showUseLastSuccessfulPrebuild, setShowUseLastSuccessfulPrebuild] = useState(false); useEffect(() => { if (!user) return; @@ -41,7 +38,6 @@ const FeatureFlagContextProvider: React.FC = ({ children }) => { const featureFlags: FeatureFlagConfig = { persistent_volume_claim: { defaultValue: true, setter: setShowPersistentVolumeClaimUI }, usage_view: { defaultValue: false, setter: setShowUsageView }, - showUseLastSuccessfulPrebuild: { defaultValue: false, setter: setShowUseLastSuccessfulPrebuild }, }; for (const [flagName, config] of Object.entries(featureFlags)) { if (teams) { @@ -73,9 +69,7 @@ const FeatureFlagContextProvider: React.FC = ({ children }) => { }, [user, teams, team, project]); return ( - + {children} ); diff --git a/components/dashboard/src/start/CreateWorkspace.tsx b/components/dashboard/src/start/CreateWorkspace.tsx index 53286c474094cd..72b388af026e38 100644 --- a/components/dashboard/src/start/CreateWorkspace.tsx +++ b/components/dashboard/src/start/CreateWorkspace.tsx @@ -29,7 +29,6 @@ import { BillingAccountSelector } from "../components/BillingAccountSelector"; import { AttributionId } from "@gitpod/gitpod-protocol/lib/attribution"; import { TeamsContext } from "../teams/teams-context"; import Alert from "../components/Alert"; -import { FeatureFlagContext } from "../contexts/FeatureFlagContext"; export interface CreateWorkspaceProps { contextUrl: string; @@ -542,7 +541,6 @@ interface RunningPrebuildViewProps { function RunningPrebuildView(props: RunningPrebuildViewProps) { const workspaceId = props.runningPrebuild.workspaceID; - const { showUseLastSuccessfulPrebuild } = useContext(FeatureFlagContext); useEffect(() => { const disposables = new DisposableCollection(); @@ -571,14 +569,12 @@ function RunningPrebuildView(props: RunningPrebuildViewProps) { {/* TODO(gpl) Copied around in Start-/CreateWorkspace. This should properly go somewhere central. */}
- {showUseLastSuccessfulPrebuild && ( - - )} + From 7514e9a21984b0ffc851c09c73ba80a24503d6ec Mon Sep 17 00:00:00 2001 From: Jan Keromnes Date: Wed, 19 Oct 2022 11:38:53 +0000 Subject: [PATCH 3/3] Revert "[server][dashboard] Implement a 'Use Last Successful Prebuild' workspace creation mode" This reverts commit 5687600aab763adc2562b31163dd337c9332aa4e. --- .../dashboard/src/start/CreateWorkspace.tsx | 10 - components/gitpod-protocol/src/protocol.ts | 9 +- components/server/ee/src/container-module.ts | 2 - .../incremental-prebuilds-service.ts | 175 ----------------- .../ee/src/prebuilds/prebuild-manager.ts | 41 ++-- .../ee/src/workspace/gitpod-server-impl.ts | 10 - .../ee/src/workspace/workspace-factory.ts | 181 ++++++++++++++---- 7 files changed, 174 insertions(+), 254 deletions(-) delete mode 100644 components/server/ee/src/prebuilds/incremental-prebuilds-service.ts diff --git a/components/dashboard/src/start/CreateWorkspace.tsx b/components/dashboard/src/start/CreateWorkspace.tsx index 72b388af026e38..6da718bef90cc7 100644 --- a/components/dashboard/src/start/CreateWorkspace.tsx +++ b/components/dashboard/src/start/CreateWorkspace.tsx @@ -269,9 +269,6 @@ export default class CreateWorkspace extends React.Component - this.createWorkspace(CreateWorkspaceMode.UseLastSuccessfulPrebuild) - } onIgnorePrebuild={() => this.createWorkspace(CreateWorkspaceMode.ForceNew)} onPrebuildSucceeded={() => this.createWorkspace(CreateWorkspaceMode.UsePrebuild)} /> @@ -534,7 +531,6 @@ interface RunningPrebuildViewProps { starting: RunningWorkspacePrebuildStarting; sameCluster: boolean; }; - onUseLastSuccessfulPrebuild: () => void; onIgnorePrebuild: () => void; onPrebuildSucceeded: () => void; } @@ -569,12 +565,6 @@ function RunningPrebuildView(props: RunningPrebuildViewProps) { {/* TODO(gpl) Copied around in Start-/CreateWorkspace. This should properly go somewhere central. */}
- diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 2c98248ebcba4c..3e97767421e660 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -1111,16 +1111,13 @@ export namespace SnapshotContext { } } -export interface WithCommitHistory { +export interface StartPrebuildContext extends WorkspaceContext { + actual: WorkspaceContext; commitHistory?: string[]; additionalRepositoryCommitHistories?: { cloneUrl: string; commitHistory: string[]; }[]; -} - -export interface StartPrebuildContext extends WorkspaceContext, WithCommitHistory { - actual: WorkspaceContext; project?: Project; branch?: string; } @@ -1385,8 +1382,6 @@ export enum CreateWorkspaceMode { UsePrebuild = "use-prebuild", // SelectIfRunning returns a list of currently running workspaces for the context URL if there are any, otherwise falls back to Default mode SelectIfRunning = "select-if-running", - // UseLastSuccessfulPrebuild returns ... - UseLastSuccessfulPrebuild = "use-last-successful-prebuild", } export namespace WorkspaceCreationResult { diff --git a/components/server/ee/src/container-module.ts b/components/server/ee/src/container-module.ts index 7dce1de91e1912..9581c434069311 100644 --- a/components/server/ee/src/container-module.ts +++ b/components/server/ee/src/container-module.ts @@ -22,7 +22,6 @@ import { PrebuildStatusMaintainer } from "./prebuilds/prebuilt-status-maintainer import { GitLabApp } from "./prebuilds/gitlab-app"; import { BitbucketApp } from "./prebuilds/bitbucket-app"; import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app"; -import { IncrementalPrebuildsService } from "./prebuilds/incremental-prebuilds-service"; import { IPrefixContextParser } from "../../src/workspace/context-parser"; import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser"; import { WorkspaceFactory } from "../../src/workspace/workspace-factory"; @@ -84,7 +83,6 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is bind(BitbucketAppSupport).toSelf().inSingletonScope(); bind(GitHubEnterpriseApp).toSelf().inSingletonScope(); bind(BitbucketServerApp).toSelf().inSingletonScope(); - bind(IncrementalPrebuildsService).toSelf().inSingletonScope(); bind(UserCounter).toSelf().inSingletonScope(); diff --git a/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts b/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts deleted file mode 100644 index 84fbeffdefb7e5..00000000000000 --- a/components/server/ee/src/prebuilds/incremental-prebuilds-service.ts +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright (c) 2022 Gitpod GmbH. All rights reserved. - * Licensed under the GNU Affero General Public License (AGPL). - * See License-AGPL.txt in the project root for license information. - */ - -import { inject, injectable } from "inversify"; -import { - CommitContext, - PrebuiltWorkspace, - TaskConfig, - User, - Workspace, - WorkspaceConfig, - WorkspaceImageSource, -} from "@gitpod/gitpod-protocol"; -import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; -import { WithCommitHistory } from "@gitpod/gitpod-protocol/src/protocol"; -import { WorkspaceDB } from "@gitpod/gitpod-db/lib"; -import { Config } from "../../../src/config"; -import { ConfigProvider } from "../../../src/workspace/config-provider"; -import { HostContextProvider } from "../../../src/auth/host-context-provider"; -import { ImageSourceProvider } from "../../../src/workspace/image-source-provider"; - -@injectable() -export class IncrementalPrebuildsService { - @inject(Config) protected readonly config: Config; - @inject(ConfigProvider) protected readonly configProvider: ConfigProvider; - @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; - @inject(ImageSourceProvider) protected readonly imageSourceProvider: ImageSourceProvider; - @inject(WorkspaceDB) protected readonly workspaceDB: WorkspaceDB; - - public async getCommitHistoryForContext(context: CommitContext, user: User): Promise { - const maxDepth = this.config.incrementalPrebuilds.commitHistory; - const hostContext = this.hostContextProvider.get(context.repository.host); - const repoProvider = hostContext?.services?.repositoryProvider; - if (!repoProvider) { - return {}; - } - const history: WithCommitHistory = {}; - history.commitHistory = await repoProvider.getCommitHistory( - user, - context.repository.owner, - context.repository.name, - context.revision, - maxDepth, - ); - if (context.additionalRepositoryCheckoutInfo && context.additionalRepositoryCheckoutInfo.length > 0) { - const histories = context.additionalRepositoryCheckoutInfo.map(async (info) => { - const commitHistory = await repoProvider.getCommitHistory( - user, - info.repository.owner, - info.repository.name, - info.revision, - maxDepth, - ); - return { - cloneUrl: info.repository.cloneUrl, - commitHistory, - }; - }); - history.additionalRepositoryCommitHistories = await Promise.all(histories); - } - return history; - } - - public async findGoodBaseForIncrementalBuild( - context: CommitContext, - history: WithCommitHistory, - user: User, - ): Promise { - if (!history.commitHistory || history.commitHistory.length < 1) { - return; - } - - const { config } = await this.configProvider.fetchConfig({}, user, context); - const imageSource = await this.imageSourceProvider.getImageSource({}, user, context, config); - - // Note: This query returns only not-garbage-collected prebuilds in order to reduce cardinality - // (e.g., at the time of writing, the Gitpod repository has 16K+ prebuilds, but only ~300 not-garbage-collected) - const recentPrebuilds = await this.workspaceDB.findPrebuildsWithWorkpace(context.repository.cloneUrl); - for (const recentPrebuild of recentPrebuilds) { - if ( - await this.isGoodBaseforIncrementalBuild( - history, - config, - imageSource, - recentPrebuild.prebuild, - recentPrebuild.workspace, - ) - ) { - return recentPrebuild.prebuild; - } - } - } - - protected async isGoodBaseforIncrementalBuild( - history: WithCommitHistory, - config: WorkspaceConfig, - imageSource: WorkspaceImageSource, - candidatePrebuild: PrebuiltWorkspace, - candidateWorkspace: Workspace, - ): Promise { - if (!history.commitHistory || history.commitHistory.length === 0) { - return false; - } - if (!CommitContext.is(candidateWorkspace.context)) { - return false; - } - - // we are only considering available prebuilds - if (candidatePrebuild.state !== "available") { - return false; - } - - // we are only considering full prebuilds - if (!!candidateWorkspace.basedOnPrebuildId) { - return false; - } - - if ( - candidateWorkspace.context.additionalRepositoryCheckoutInfo?.length !== - history.additionalRepositoryCommitHistories?.length - ) { - // different number of repos - return false; - } - - const candidateCtx = candidateWorkspace.context; - if (!history.commitHistory.some((sha) => sha === candidateCtx.revision)) { - return false; - } - - // check the commits are included in the commit history - for (const subRepo of candidateWorkspace.context.additionalRepositoryCheckoutInfo || []) { - const matchIngRepo = history.additionalRepositoryCommitHistories?.find( - (repo) => repo.cloneUrl === subRepo.repository.cloneUrl, - ); - if (!matchIngRepo || !matchIngRepo.commitHistory.some((sha) => sha === subRepo.revision)) { - return false; - } - } - - // ensure the image source hasn't changed (skips older images) - if (JSON.stringify(imageSource) !== JSON.stringify(candidateWorkspace.imageSource)) { - log.debug(`Skipping parent prebuild: Outdated image`, { - imageSource, - parentImageSource: candidateWorkspace.imageSource, - }); - return false; - } - - // ensure the tasks haven't changed - const filterPrebuildTasks = (tasks: TaskConfig[] = []) => - tasks - .map((task) => - Object.keys(task) - .filter((key) => ["before", "init", "prebuild"].includes(key)) - // @ts-ignore - .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}), - ) - .filter((task) => Object.keys(task).length > 0); - const prebuildTasks = filterPrebuildTasks(config.tasks); - const parentPrebuildTasks = filterPrebuildTasks(candidateWorkspace.config.tasks); - if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) { - log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, { - prebuildTasks, - parentPrebuildTasks, - }); - return false; - } - - return true; - } -} diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index 8d161ea8546b8d..ec357a00e636f8 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -33,7 +33,6 @@ import { inject, injectable } from "inversify"; import * as opentracing from "opentracing"; import { StopWorkspacePolicy } from "@gitpod/ws-manager/lib"; import { error } from "console"; -import { IncrementalPrebuildsService } from "./incremental-prebuilds-service"; export class WorkspaceRunningError extends Error { constructor(msg: string, public instance: WorkspaceInstance) { @@ -60,7 +59,6 @@ export class PrebuildManager { @inject(ConfigProvider) protected readonly configProvider: ConfigProvider; @inject(Config) protected readonly config: Config; @inject(ProjectsService) protected readonly projectService: ProjectsService; - @inject(IncrementalPrebuildsService) protected readonly incrementalPrebuildsService: IncrementalPrebuildsService; async abortPrebuildsForBranch(ctx: TraceContext, project: Project, user: User, branch: string): Promise { const span = TraceContext.startSpan("abortPrebuildsForBranch", ctx); @@ -174,15 +172,36 @@ export class PrebuildManager { }; if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) { - // We store the commit histories in the `StartPrebuildContext` in order to pass them down to - // `WorkspaceFactoryEE.createForStartPrebuild`. - const { commitHistory, additionalRepositoryCommitHistories } = - await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); - if (commitHistory) { - prebuildContext.commitHistory = commitHistory; - } - if (additionalRepositoryCommitHistories) { - prebuildContext.additionalRepositoryCommitHistories = additionalRepositoryCommitHistories; + const maxDepth = this.config.incrementalPrebuilds.commitHistory; + const hostContext = this.hostContextProvider.get(context.repository.host); + const repoProvider = hostContext?.services?.repositoryProvider; + if (repoProvider) { + prebuildContext.commitHistory = await repoProvider.getCommitHistory( + user, + context.repository.owner, + context.repository.name, + context.revision, + maxDepth, + ); + if ( + context.additionalRepositoryCheckoutInfo && + context.additionalRepositoryCheckoutInfo.length > 0 + ) { + const histories = context.additionalRepositoryCheckoutInfo.map(async (info) => { + const commitHistory = await repoProvider.getCommitHistory( + user, + info.repository.owner, + info.repository.name, + info.revision, + maxDepth, + ); + return { + cloneUrl: info.repository.cloneUrl, + commitHistory, + }; + }); + prebuildContext.additionalRepositoryCommitHistories = await Promise.all(histories); + } } } diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index eb8526110d8418..f89b357c92f97d 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -118,12 +118,10 @@ import { BillingModes } from "../billing/billing-mode"; import { UsageServiceDefinition } from "@gitpod/usage-api/lib/usage/v1/usage.pb"; import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server"; import { BillingServiceClient, BillingServiceDefinition } from "@gitpod/usage-api/lib/usage/v1/billing.pb"; -import { IncrementalPrebuildsService } from "../prebuilds/incremental-prebuilds-service"; @injectable() export class GitpodServerEEImpl extends GitpodServerImpl { @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager; - @inject(IncrementalPrebuildsService) protected readonly incrementalPrebuildsService: IncrementalPrebuildsService; @inject(LicenseDB) protected readonly licenseDB: LicenseDB; @inject(LicenseKeySource) protected readonly licenseKeySource: LicenseKeySource; @@ -986,14 +984,6 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace }; log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); - if (prebuiltWorkspace?.state !== "available" && mode === CreateWorkspaceMode.UseLastSuccessfulPrebuild) { - const history = await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); - prebuiltWorkspace = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( - context, - history, - user, - ); - } if (!prebuiltWorkspace) { return; } diff --git a/components/server/ee/src/workspace/workspace-factory.ts b/components/server/ee/src/workspace/workspace-factory.ts index 8596054fd351b4..ca195c502ad5e6 100644 --- a/components/server/ee/src/workspace/workspace-factory.ts +++ b/components/server/ee/src/workspace/workspace-factory.ts @@ -17,6 +17,10 @@ import { WorkspaceContext, WithSnapshot, WithPrebuild, + TaskConfig, + PrebuiltWorkspace, + WorkspaceConfig, + WorkspaceImageSource, OpenPrebuildContext, } from "@gitpod/gitpod-protocol"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; @@ -31,7 +35,6 @@ import { increasePrebuildsStartedCounter } from "../../../src/prometheus-metrics import { DeepPartial } from "@gitpod/gitpod-protocol/lib/util/deep-partial"; import { EntitlementService } from "../../../src/billing/entitlement-service"; import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server"; -import { IncrementalPrebuildsService } from "../prebuilds/incremental-prebuilds-service"; @injectable() export class WorkspaceFactoryEE extends WorkspaceFactory { @@ -39,7 +42,6 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; @inject(UserCounter) protected readonly userCounter: UserCounter; @inject(EntitlementService) protected readonly entitlementService: EntitlementService; - @inject(IncrementalPrebuildsService) protected readonly incrementalPrebuildsService: IncrementalPrebuildsService; @inject(UserDB) protected readonly userDB: UserDB; @@ -116,46 +118,68 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { await assertNoPrebuildIsRunningForSameCommit(); const { config } = await this.configProvider.fetchConfig({ span }, user, context.actual); + const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config); - // If an incremental prebuild was requested, see if we can find a recent prebuild to act as a base. + // Walk back the last prebuilds and check if they are valid ancestor. let ws; - const recentPrebuild = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( - commitContext, - context, - user, - ); - if (recentPrebuild) { - const loggedContext = filterForLogging(context); - log.info({ userId: user.id }, "Using incremental prebuild base", { - basePrebuildId: recentPrebuild.id, - context: loggedContext, - }); - - const incrementalPrebuildContext: PrebuiltWorkspaceContext = { - title: `Incremental prebuild of "${commitContext.title}"`, - originalContext: commitContext, - prebuiltWorkspace: recentPrebuild, - }; - - // repeated assertion on prebuilds triggered for same commit here, in order to - // reduce likelihood of duplicates if for instance handled by two different - // server pods. - await assertNoPrebuildIsRunningForSameCommit(); + if (context.commitHistory && context.commitHistory.length > 0) { + // Note: This query returns only not-garbage-collected prebuilds in order to reduce cardinality + // (e.g., at the time of writing, the Gitpod repository has 16K+ prebuilds, but only ~300 not-garbage-collected) + const recentPrebuilds = await this.db + .trace({ span }) + .findPrebuildsWithWorkpace(commitContext.repository.cloneUrl); - ws = await this.createForPrebuiltWorkspace( - { span }, - user, - incrementalPrebuildContext, - normalizedContextURL, - ); - // Overwrite the config from the parent prebuild: - // `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild. - // Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now. - // However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...) - // To fix this, we overwrite the new prebuild's config with the most-recently fetched config. - // See also: https://github.com/gitpod-io/gitpod/issues/7475 - //TODO(sven) doing side effects on objects back and forth is complicated and error-prone. We should rather make sure we pass in the config when creating the prebuiltWorkspace. - ws.config = config; + const loggedContext = filterForLogging(context); + for (const recentPrebuild of recentPrebuilds) { + if ( + !(await this.isGoodBaseforIncrementalPrebuild( + context, + config, + imageSource, + recentPrebuild.prebuild, + recentPrebuild.workspace, + )) + ) { + log.debug({ userId: user.id }, "Not using incremental prebuild base", { + candidatePrebuildId: recentPrebuild.prebuild.id, + context: loggedContext, + }); + continue; + } + + log.info({ userId: user.id }, "Using incremental prebuild base", { + basePrebuildId: recentPrebuild.prebuild.id, + context: loggedContext, + }); + + const incrementalPrebuildContext: PrebuiltWorkspaceContext = { + title: `Incremental prebuild of "${commitContext.title}"`, + originalContext: commitContext, + prebuiltWorkspace: recentPrebuild.prebuild, + }; + + // repeated assertion on prebuilds triggered for same commit here, in order to + // reduce likelihood of duplicates if for instance handled by two different + // server pods. + await assertNoPrebuildIsRunningForSameCommit(); + + ws = await this.createForPrebuiltWorkspace( + { span }, + user, + incrementalPrebuildContext, + normalizedContextURL, + ); + // Overwrite the config from the parent prebuild: + // `createForPrebuiltWorkspace` 1:1 copies the config from the parent prebuild. + // Above, we've made sure that the parent's prebuild tasks (before/init/prebuild) are still the same as now. + // However, other non-prebuild config items might be outdated (e.g. any command task, VS Code extension, ...) + // To fix this, we overwrite the new prebuild's config with the most-recently fetched config. + // See also: https://github.com/gitpod-io/gitpod/issues/7475 + //TODO(sven) doing side effects on objects back and forth is complicated and error-prone. We should rather make sure we pass in the config when creating the prebuiltWorkspace. + ws.config = config; + + break; + } } // repeated assertion on prebuilds triggered for same commit here, in order to @@ -213,6 +237,85 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { } } + private async isGoodBaseforIncrementalPrebuild( + context: StartPrebuildContext, + config: WorkspaceConfig, + imageSource: WorkspaceImageSource, + candidatePrebuild: PrebuiltWorkspace, + candidate: Workspace, + ): Promise { + if (!context.commitHistory || context.commitHistory.length === 0) { + return false; + } + if (!CommitContext.is(candidate.context)) { + return false; + } + + // we are only considering available prebuilds + if (candidatePrebuild.state !== "available") { + return false; + } + + // we are only considering full prebuilds + if (!!candidate.basedOnPrebuildId) { + return false; + } + + const candidateCtx = candidate.context; + if ( + candidateCtx.additionalRepositoryCheckoutInfo?.length !== + context.additionalRepositoryCommitHistories?.length + ) { + // different number of repos + return false; + } + + if (!context.commitHistory.some((sha) => sha === candidateCtx.revision)) { + return false; + } + + // check the commits are included in the commit history + for (const subRepo of candidateCtx.additionalRepositoryCheckoutInfo || []) { + const matchIngRepo = context.additionalRepositoryCommitHistories?.find( + (repo) => repo.cloneUrl === subRepo.repository.cloneUrl, + ); + if (!matchIngRepo || !matchIngRepo.commitHistory.some((sha) => sha === subRepo.revision)) { + return false; + } + } + + // ensure the image source hasn't changed (skips older images) + if (JSON.stringify(imageSource) !== JSON.stringify(candidate.imageSource)) { + log.debug(`Skipping parent prebuild: Outdated image`, { + imageSource, + parentImageSource: candidate.imageSource, + }); + return false; + } + + // ensure the tasks haven't changed + const filterPrebuildTasks = (tasks: TaskConfig[] = []) => + tasks + .map((task) => + Object.keys(task) + .filter((key) => ["before", "init", "prebuild"].includes(key)) + // @ts-ignore + .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}), + ) + .filter((task) => Object.keys(task).length > 0); + const prebuildTasks = filterPrebuildTasks(config.tasks); + const parentPrebuildTasks = filterPrebuildTasks(candidate.config.tasks); + if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) { + log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, { + prebuildTasks, + parentPrebuildTasks, + }); + return false; + } + + return true; + } + protected async createForPrebuiltWorkspace( ctx: TraceContext, user: User,