Re-add "Use Last Successful Prebuild"

components/dashboard/src/contexts/FeatureFlagContext.tsx

@@ -18,9 +18,11 @@ interface FeatureFlagConfig {
 const FeatureFlagContext = createContext<{
     showPersistentVolumeClaimUI: boolean;
     showUsageView: boolean;
+    showUseLastSuccessfulPrebuild: boolean;
 }>({
     showPersistentVolumeClaimUI: false,
     showUsageView: false,
+    showUseLastSuccessfulPrebuild: false,
 });
 
 const FeatureFlagContextProvider: React.FC = ({ children }) => {
@@ -31,13 +33,15 @@ const FeatureFlagContextProvider: React.FC = ({ children }) => {
     const team = getCurrentTeam(location, teams);
     const [showPersistentVolumeClaimUI, setShowPersistentVolumeClaimUI] = useState<boolean>(false);
     const [showUsageView, setShowUsageView] = useState<boolean>(false);
+    const [showUseLastSuccessfulPrebuild, setShowUseLastSuccessfulPrebuild] = useState<boolean>(false);
 
     useEffect(() => {
         if (!user) return;
         (async () => {
             const featureFlags: FeatureFlagConfig = {
                 persistent_volume_claim: { defaultValue: true, setter: setShowPersistentVolumeClaimUI },
                 usage_view: { defaultValue: false, setter: setShowUsageView },
+                showUseLastSuccessfulPrebuild: { defaultValue: false, setter: setShowUseLastSuccessfulPrebuild },
             };
             for (const [flagName, config] of Object.entries(featureFlags)) {
                 if (teams) {
@@ -69,7 +73,9 @@ const FeatureFlagContextProvider: React.FC = ({ children }) => {
     }, [user, teams, team, project]);
 
     return (
-        <FeatureFlagContext.Provider value={{ showPersistentVolumeClaimUI, showUsageView }}>
+        <FeatureFlagContext.Provider
+            value={{ showPersistentVolumeClaimUI, showUsageView, showUseLastSuccessfulPrebuild }}
+        >
             {children}
         </FeatureFlagContext.Provider>
     );

components/dashboard/src/start/CreateWorkspace.tsx

@@ -29,6 +29,7 @@ import { BillingAccountSelector } from "../components/BillingAccountSelector";
 import { AttributionId } from "@gitpod/gitpod-protocol/lib/attribution";
 import { TeamsContext } from "../teams/teams-context";
 import Alert from "../components/Alert";
+import { FeatureFlagContext } from "../contexts/FeatureFlagContext";
 
 export interface CreateWorkspaceProps {
     contextUrl: string;
@@ -269,6 +270,9 @@ export default class CreateWorkspace extends React.Component<CreateWorkspaceProp
             return (
                 <RunningPrebuildView
                     runningPrebuild={result.runningWorkspacePrebuild}
+                    onUseLastSuccessfulPrebuild={() =>
+                        this.createWorkspace(CreateWorkspaceMode.UseLastSuccessfulPrebuild)
+                    }
                     onIgnorePrebuild={() => this.createWorkspace(CreateWorkspaceMode.ForceNew)}
                     onPrebuildSucceeded={() => this.createWorkspace(CreateWorkspaceMode.UsePrebuild)}
                 />
@@ -531,12 +535,14 @@ interface RunningPrebuildViewProps {
         starting: RunningWorkspacePrebuildStarting;
         sameCluster: boolean;
     };
+    onUseLastSuccessfulPrebuild: () => void;
     onIgnorePrebuild: () => void;
     onPrebuildSucceeded: () => void;
 }
 
 function RunningPrebuildView(props: RunningPrebuildViewProps) {
     const workspaceId = props.runningPrebuild.workspaceID;
+    const { showUseLastSuccessfulPrebuild } = useContext(FeatureFlagContext);
 
     useEffect(() => {
         const disposables = new DisposableCollection();
@@ -565,6 +571,14 @@ function RunningPrebuildView(props: RunningPrebuildViewProps) {
             {/* TODO(gpl) Copied around in Start-/CreateWorkspace. This should properly go somewhere central. */}
             <div className="h-full mt-6 w-11/12 lg:w-3/5">
                 <PrebuildLogs workspaceId={workspaceId} onIgnorePrebuild={props.onIgnorePrebuild}>
+                    {showUseLastSuccessfulPrebuild && (
+                        <button
+                            className="secondary"
+                            onClick={() => props.onUseLastSuccessfulPrebuild && props.onUseLastSuccessfulPrebuild()}
+                        >
+                            Use Last Successful Prebuild
+                        </button>
+                    )}
                     <button className="secondary" onClick={() => props.onIgnorePrebuild && props.onIgnorePrebuild()}>
                         Skip Prebuild
                     </button>

components/gitpod-db/src/accounting-db.spec.db.ts

@@ -26,7 +26,7 @@ export class AccountingDBSpec {
     db: AccountingDB;
     queryRunner: QueryRunner;
 
-    @timeout(10000)
+    @timeout(30000)
     async before() {
         const connection = await this.typeORM.getConnection();
         const manager = connection.manager;

components/gitpod-protocol/src/protocol.ts

@@ -1111,13 +1111,16 @@ export namespace SnapshotContext {
     }
 }
 
-export interface StartPrebuildContext extends WorkspaceContext {
-    actual: WorkspaceContext;
+export interface WithCommitHistory {
     commitHistory?: string[];
     additionalRepositoryCommitHistories?: {
         cloneUrl: string;
         commitHistory: string[];
     }[];
+}
+
+export interface StartPrebuildContext extends WorkspaceContext, WithCommitHistory {
+    actual: WorkspaceContext;
     project?: Project;
     branch?: string;
 }
@@ -1382,6 +1385,8 @@ export enum CreateWorkspaceMode {
     UsePrebuild = "use-prebuild",
     // SelectIfRunning returns a list of currently running workspaces for the context URL if there are any, otherwise falls back to Default mode
     SelectIfRunning = "select-if-running",
+    // UseLastSuccessfulPrebuild returns ...
+    UseLastSuccessfulPrebuild = "use-last-successful-prebuild",
 }
 
 export namespace WorkspaceCreationResult {

components/server/ee/src/container-module.ts

@@ -22,6 +22,7 @@ import { PrebuildStatusMaintainer } from "./prebuilds/prebuilt-status-maintainer
 import { GitLabApp } from "./prebuilds/gitlab-app";
 import { BitbucketApp } from "./prebuilds/bitbucket-app";
 import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app";
+import { IncrementalPrebuildsService } from "./prebuilds/incremental-prebuilds-service";
 import { IPrefixContextParser } from "../../src/workspace/context-parser";
 import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser";
 import { WorkspaceFactory } from "../../src/workspace/workspace-factory";
@@ -83,6 +84,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
     bind(BitbucketAppSupport).toSelf().inSingletonScope();
     bind(GitHubEnterpriseApp).toSelf().inSingletonScope();
     bind(BitbucketServerApp).toSelf().inSingletonScope();
+    bind(IncrementalPrebuildsService).toSelf().inSingletonScope();
 
     bind(UserCounter).toSelf().inSingletonScope();
 

components/server/ee/src/prebuilds/incremental-prebuilds-service.ts

@@ -0,0 +1,175 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { inject, injectable } from "inversify";
+import {
+    CommitContext,
+    PrebuiltWorkspace,
+    TaskConfig,
+    User,
+    Workspace,
+    WorkspaceConfig,
+    WorkspaceImageSource,
+} from "@gitpod/gitpod-protocol";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { WithCommitHistory } from "@gitpod/gitpod-protocol/src/protocol";
+import { WorkspaceDB } from "@gitpod/gitpod-db/lib";
+import { Config } from "../../../src/config";
+import { ConfigProvider } from "../../../src/workspace/config-provider";
+import { HostContextProvider } from "../../../src/auth/host-context-provider";
+import { ImageSourceProvider } from "../../../src/workspace/image-source-provider";
+
+@injectable()
+export class IncrementalPrebuildsService {
+    @inject(Config) protected readonly config: Config;
+    @inject(ConfigProvider) protected readonly configProvider: ConfigProvider;
+    @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
+    @inject(ImageSourceProvider) protected readonly imageSourceProvider: ImageSourceProvider;
+    @inject(WorkspaceDB) protected readonly workspaceDB: WorkspaceDB;
+
+    public async getCommitHistoryForContext(context: CommitContext, user: User): Promise<WithCommitHistory> {
+        const maxDepth = this.config.incrementalPrebuilds.commitHistory;
+        const hostContext = this.hostContextProvider.get(context.repository.host);
+        const repoProvider = hostContext?.services?.repositoryProvider;
+        if (!repoProvider) {
+            return {};
+        }
+        const history: WithCommitHistory = {};
+        history.commitHistory = await repoProvider.getCommitHistory(
+            user,
+            context.repository.owner,
+            context.repository.name,
+            context.revision,
+            maxDepth,
+        );
+        if (context.additionalRepositoryCheckoutInfo && context.additionalRepositoryCheckoutInfo.length > 0) {
+            const histories = context.additionalRepositoryCheckoutInfo.map(async (info) => {
+                const commitHistory = await repoProvider.getCommitHistory(
+                    user,
+                    info.repository.owner,
+                    info.repository.name,
+                    info.revision,
+                    maxDepth,
+                );
+                return {
+                    cloneUrl: info.repository.cloneUrl,
+                    commitHistory,
+                };
+            });
+            history.additionalRepositoryCommitHistories = await Promise.all(histories);
+        }
+        return history;
+    }
+
+    public async findGoodBaseForIncrementalBuild(
+        context: CommitContext,
+        config: WorkspaceConfig,
+        history: WithCommitHistory,
+        user: User,
+    ): Promise<PrebuiltWorkspace | undefined> {
+        if (!history.commitHistory || history.commitHistory.length < 1) {
+            return;
+        }
+
+        const imageSource = await this.imageSourceProvider.getImageSource({}, user, context, config);
+
+        // Note: This query returns only not-garbage-collected prebuilds in order to reduce cardinality
+        // (e.g., at the time of writing, the Gitpod repository has 16K+ prebuilds, but only ~300 not-garbage-collected)
+        const recentPrebuilds = await this.workspaceDB.findPrebuildsWithWorkpace(context.repository.cloneUrl);
+        for (const recentPrebuild of recentPrebuilds) {
+            if (
+                await this.isGoodBaseforIncrementalBuild(
+                    history,
+                    config,
+                    imageSource,
+                    recentPrebuild.prebuild,
+                    recentPrebuild.workspace,
+                )
+            ) {
+                return recentPrebuild.prebuild;
+            }
+        }
+    }
+
+    protected async isGoodBaseforIncrementalBuild(
+        history: WithCommitHistory,
+        config: WorkspaceConfig,
+        imageSource: WorkspaceImageSource,
+        candidatePrebuild: PrebuiltWorkspace,
+        candidateWorkspace: Workspace,
+    ): Promise<boolean> {
+        if (!history.commitHistory || history.commitHistory.length === 0) {
+            return false;
+        }
+        if (!CommitContext.is(candidateWorkspace.context)) {
+            return false;
+        }
+
+        // we are only considering available prebuilds
+        if (candidatePrebuild.state !== "available") {
+            return false;
+        }
+
+        // we are only considering full prebuilds
+        if (!!candidateWorkspace.basedOnPrebuildId) {
+            return false;
+        }
+
+        if (
+            candidateWorkspace.context.additionalRepositoryCheckoutInfo?.length !==
+            history.additionalRepositoryCommitHistories?.length
+        ) {
+            // different number of repos
+            return false;
+        }
+
+        const candidateCtx = candidateWorkspace.context;
+        if (!history.commitHistory.some((sha) => sha === candidateCtx.revision)) {
+            return false;
+        }
+
+        // check the commits are included in the commit history
+        for (const subRepo of candidateWorkspace.context.additionalRepositoryCheckoutInfo || []) {
+            const matchIngRepo = history.additionalRepositoryCommitHistories?.find(
+                (repo) => repo.cloneUrl === subRepo.repository.cloneUrl,
+            );
+            if (!matchIngRepo || !matchIngRepo.commitHistory.some((sha) => sha === subRepo.revision)) {
+                return false;
+            }
+        }
+
+        // ensure the image source hasn't changed (skips older images)
+        if (JSON.stringify(imageSource) !== JSON.stringify(candidateWorkspace.imageSource)) {
+            log.debug(`Skipping parent prebuild: Outdated image`, {
+                imageSource,
+                parentImageSource: candidateWorkspace.imageSource,
+            });
+            return false;
+        }
+
+        // ensure the tasks haven't changed
+        const filterPrebuildTasks = (tasks: TaskConfig[] = []) =>
+            tasks
+                .map((task) =>
+                    Object.keys(task)
+                        .filter((key) => ["before", "init", "prebuild"].includes(key))
+                        // @ts-ignore
+                        .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}),
+                )
+                .filter((task) => Object.keys(task).length > 0);
+        const prebuildTasks = filterPrebuildTasks(config.tasks);
+        const parentPrebuildTasks = filterPrebuildTasks(candidateWorkspace.config.tasks);
+        if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
+            log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
+                prebuildTasks,
+                parentPrebuildTasks,
+            });
+            return false;
+        }
+
+        return true;
+    }
+}

components/server/ee/src/prebuilds/prebuild-manager.ts

@@ -33,6 +33,7 @@ import { inject, injectable } from "inversify";
 import * as opentracing from "opentracing";
 import { StopWorkspacePolicy } from "@gitpod/ws-manager/lib";
 import { error } from "console";
+import { IncrementalPrebuildsService } from "./incremental-prebuilds-service";
 
 export class WorkspaceRunningError extends Error {
     constructor(msg: string, public instance: WorkspaceInstance) {
@@ -59,6 +60,7 @@ export class PrebuildManager {
     @inject(ConfigProvider) protected readonly configProvider: ConfigProvider;
     @inject(Config) protected readonly config: Config;
     @inject(ProjectsService) protected readonly projectService: ProjectsService;
+    @inject(IncrementalPrebuildsService) protected readonly incrementalPrebuildsService: IncrementalPrebuildsService;
 
     async abortPrebuildsForBranch(ctx: TraceContext, project: Project, user: User, branch: string): Promise<void> {
         const span = TraceContext.startSpan("abortPrebuildsForBranch", ctx);
@@ -172,36 +174,15 @@ export class PrebuildManager {
             };
 
             if (this.shouldPrebuildIncrementally(context.repository.cloneUrl, project)) {
-                const maxDepth = this.config.incrementalPrebuilds.commitHistory;
-                const hostContext = this.hostContextProvider.get(context.repository.host);
-                const repoProvider = hostContext?.services?.repositoryProvider;
-                if (repoProvider) {
-                    prebuildContext.commitHistory = await repoProvider.getCommitHistory(
-                        user,
-                        context.repository.owner,
-                        context.repository.name,
-                        context.revision,
-                        maxDepth,
-                    );
-                    if (
-                        context.additionalRepositoryCheckoutInfo &&
-                        context.additionalRepositoryCheckoutInfo.length > 0
-                    ) {
-                        const histories = context.additionalRepositoryCheckoutInfo.map(async (info) => {
-                            const commitHistory = await repoProvider.getCommitHistory(
-                                user,
-                                info.repository.owner,
-                                info.repository.name,
-                                info.revision,
-                                maxDepth,
-                            );
-                            return {
-                                cloneUrl: info.repository.cloneUrl,
-                                commitHistory,
-                            };
-                        });
-                        prebuildContext.additionalRepositoryCommitHistories = await Promise.all(histories);
-                    }
+                // We store the commit histories in the `StartPrebuildContext` in order to pass them down to
+                // `WorkspaceFactoryEE.createForStartPrebuild`.
+                const { commitHistory, additionalRepositoryCommitHistories } =
+                    await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user);
+                if (commitHistory) {
+                    prebuildContext.commitHistory = commitHistory;
+                }
+                if (additionalRepositoryCommitHistories) {
+                    prebuildContext.additionalRepositoryCommitHistories = additionalRepositoryCommitHistories;
                 }
             }