From feabf7202ba95e467709eb46827dd418e74a3dec Mon Sep 17 00:00:00 2001 From: Jan Keromnes Date: Fri, 14 Jan 2022 14:32:27 +0000 Subject: [PATCH] [dashboard][server] Make Project Overview page faster by pre-fetching and caching Git provider data (branches) --- components/gitpod-db/src/project-db.ts | 2 + components/gitpod-db/src/tables.ts | 6 +++ .../gitpod-db/src/typeorm/deleted-entry-gc.ts | 1 + .../typeorm/entity/db-prebuild-info-entry.ts | 2 +- .../src/typeorm/entity/db-project-info.ts | 43 +++++++++++++++++++ .../migration/1642497869312-ProjectInfo.ts | 18 ++++++++ .../gitpod-db/src/typeorm/project-db-impl.ts | 26 +++++++++++ .../src/teams-projects-protocol.ts | 8 +++- .../server/src/projects/projects-service.ts | 26 +++++++++-- .../src/workspace/gitpod-server-impl.ts | 2 +- 10 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 components/gitpod-db/src/typeorm/entity/db-project-info.ts create mode 100644 components/gitpod-db/src/typeorm/migration/1642497869312-ProjectInfo.ts diff --git a/components/gitpod-db/src/project-db.ts b/components/gitpod-db/src/project-db.ts index 93834545d3b736..13db28f1fbdcb6 100644 --- a/components/gitpod-db/src/project-db.ts +++ b/components/gitpod-db/src/project-db.ts @@ -21,4 +21,6 @@ export interface ProjectDB { getProjectEnvironmentVariableById(variableId: string): Promise; deleteProjectEnvironmentVariable(variableId: string): Promise; getProjectEnvironmentVariableValues(envVars: ProjectEnvVar[]): Promise; + findCachedProjectOverview(projectId: string): Promise; + storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise; } diff --git a/components/gitpod-db/src/tables.ts b/components/gitpod-db/src/tables.ts index 705ab415342ef0..6d61e77a5cd303 100644 --- a/components/gitpod-db/src/tables.ts +++ b/components/gitpod-db/src/tables.ts @@ -251,6 +251,12 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider deletionColumn: 'deleted', timeColumn: '_lastModified', }, + { + name: 'd_b_project_info', + primaryKeys: ['projectId'], + deletionColumn: 'deleted', + timeColumn: '_lastModified', + }, /** * BEWARE * diff --git a/components/gitpod-db/src/typeorm/deleted-entry-gc.ts b/components/gitpod-db/src/typeorm/deleted-entry-gc.ts index abd9ed73f281ad..86ebfdd2b43bf1 100644 --- a/components/gitpod-db/src/typeorm/deleted-entry-gc.ts +++ b/components/gitpod-db/src/typeorm/deleted-entry-gc.ts @@ -59,6 +59,7 @@ const tables: TableWithDeletion[] = [ { deletionColumn: "deleted", name: "d_b_prebuild_info" }, { deletionColumn: "deleted", name: "d_b_oss_allow_list" }, { deletionColumn: "deleted", name: "d_b_project_env_var" }, + { deletionColumn: "deleted", name: "d_b_project_info" }, ]; interface TableWithDeletion { diff --git a/components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts b/components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts index ea736e338fbe9a..9f9c8eaf042b9a 100644 --- a/components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts +++ b/components/gitpod-db/src/typeorm/entity/db-prebuild-info-entry.ts @@ -10,7 +10,7 @@ import { PrebuildInfo } from "@gitpod/gitpod-protocol"; import { TypeORM } from "../../typeorm/typeorm"; @Entity() -export class DBPrebuildInfo { +export class DBPrebuildInfo { @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE) prebuildId: string; diff --git a/components/gitpod-db/src/typeorm/entity/db-project-info.ts b/components/gitpod-db/src/typeorm/entity/db-project-info.ts new file mode 100644 index 00000000000000..b80a8e552c7091 --- /dev/null +++ b/components/gitpod-db/src/typeorm/entity/db-project-info.ts @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021 Gitpod GmbH. All rights reserved. + * Licensed under the Gitpod Enterprise Source Code License, + * See License.enterprise.txt in the project root folder. + */ + +import { Entity, Column, PrimaryColumn } from "typeorm"; +import { Project } from "@gitpod/gitpod-protocol"; + +import { TypeORM } from "../../typeorm/typeorm"; + +@Entity() +// on DB but not Typeorm: @Index("ind_dbsync", ["_lastModified"]) // DBSync +export class DBProjectInfo { + + @PrimaryColumn(TypeORM.UUID_COLUMN_TYPE) + projectId: string; + + @Column({ + type: 'simple-json', + transformer: (() => { + return { + to(value: any): any { + return JSON.stringify(value); + }, + from(value: any): any { + try { + const obj = JSON.parse(value); + if (Project.Overview.is(obj)) { + return obj; + } + } catch (error) { + } + } + }; + })() + }) + overview: Project.Overview; + + // This column triggers the db-sync deletion mechanism. It's not intended for public consumption. + @Column() + deleted: boolean; +} \ No newline at end of file diff --git a/components/gitpod-db/src/typeorm/migration/1642497869312-ProjectInfo.ts b/components/gitpod-db/src/typeorm/migration/1642497869312-ProjectInfo.ts new file mode 100644 index 00000000000000..93965194ff0aa3 --- /dev/null +++ b/components/gitpod-db/src/typeorm/migration/1642497869312-ProjectInfo.ts @@ -0,0 +1,18 @@ +/** + * Copyright (c) 2022 Gitpod GmbH. All rights reserved. + * Licensed under the GNU Affero General Public License (AGPL). + * See License-AGPL.txt in the project root for license information. + */ + +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class ProjectInfo1642497869312 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query("CREATE TABLE IF NOT EXISTS `d_b_project_info` ( `projectId` char(36) NOT NULL, `overview` longtext NOT NULL, `deleted` tinyint(4) NOT NULL DEFAULT '0', `_lastModified` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (`projectId`), KEY `ind_dbsync` (`_lastModified`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"); + } + + public async down(queryRunner: QueryRunner): Promise { + } + +} diff --git a/components/gitpod-db/src/typeorm/project-db-impl.ts b/components/gitpod-db/src/typeorm/project-db-impl.ts index 14d094cfc206c3..8259a114f4d217 100644 --- a/components/gitpod-db/src/typeorm/project-db-impl.ts +++ b/components/gitpod-db/src/typeorm/project-db-impl.ts @@ -13,6 +13,7 @@ import { EncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryp import { ProjectDB } from "../project-db"; import { DBProject } from "./entity/db-project"; import { DBProjectEnvVar } from "./entity/db-project-env-vars"; +import { DBProjectInfo } from "./entity/db-project-info"; function toProjectEnvVar(envVarWithValue: ProjectEnvVarWithValue): ProjectEnvVar { const envVar = { ...envVarWithValue }; @@ -37,6 +38,10 @@ export class ProjectDBImpl implements ProjectDB { return (await this.getEntityManager()).getRepository(DBProjectEnvVar); } + protected async getProjectInfoRepo(): Promise> { + return (await this.getEntityManager()).getRepository(DBProjectInfo); + } + public async findProjectById(projectId: string): Promise { const repo = await this.getRepo(); return repo.findOne({ id: projectId, markedDeleted: false }); @@ -106,6 +111,12 @@ export class ProjectDBImpl implements ProjectDB { project.markedDeleted = true; await repo.save(project); } + // Delete any additional cached infos about this project + const projectInfoRepo = await this.getProjectInfoRepo(); + const info = await projectInfoRepo.findOne({ projectId, deleted: false }); + if (info) { + await projectInfoRepo.update(projectId, { deleted: true }); + } } public async setProjectEnvironmentVariable(projectId: string, name: string, value: string, censored: boolean): Promise { @@ -164,4 +175,19 @@ export class ProjectDBImpl implements ProjectDB { const envVarsWithValues = await envVarRepo.findByIds(envVars); return envVarsWithValues; } + + public async findCachedProjectOverview(projectId: string): Promise { + const projectInfoRepo = await this.getProjectInfoRepo(); + const info = await projectInfoRepo.findOne({ projectId }); + return info?.overview; + } + + public async storeCachedProjectOverview(projectId: string, overview: Project.Overview): Promise { + const projectInfoRepo = await this.getProjectInfoRepo(); + await projectInfoRepo.save({ + projectId, + overview, + creationTime: new Date().toISOString(), + }); + } } diff --git a/components/gitpod-protocol/src/teams-projects-protocol.ts b/components/gitpod-protocol/src/teams-projects-protocol.ts index eae40284c65778..36fd67ddecd82b 100644 --- a/components/gitpod-protocol/src/teams-projects-protocol.ts +++ b/components/gitpod-protocol/src/teams-projects-protocol.ts @@ -42,7 +42,13 @@ export namespace Project { } export interface Overview { - branches: BranchDetails[] + branches: BranchDetails[]; + } + + export namespace Overview { + export function is(data?: any): data is Project.Overview { + return Array.isArray(data?.branches); + } } export interface BranchDetails { diff --git a/components/server/src/projects/projects-service.ts b/components/server/src/projects/projects-service.ts index 371393b6bba5bd..347febf68954ab 100644 --- a/components/server/src/projects/projects-service.ts +++ b/components/server/src/projects/projects-service.ts @@ -37,9 +37,25 @@ export class ProjectsService { return this.projectDB.findProjectsByCloneUrls(cloneUrls); } - async getProjectOverview(user: User, project: Project): Promise { - const branches = await this.getBranchDetails(user, project); - return { branches }; + async getProjectOverviewCached(user: User, project: Project): Promise { + // Check for a cached project overview (fast!) + const cachedPromise = this.projectDB.findCachedProjectOverview(project.id); + + // ...but also refresh the cache on every request (asynchronously / in the background) + const refreshPromise = this.getBranchDetails(user, project).then(branches => { + const overview = { branches }; + // No need to await here + this.projectDB.storeCachedProjectOverview(project.id, overview).catch(error => { + log.error(`Could not store cached project overview: ${error}`, { cloneUrl: project.cloneUrl }) + }); + return overview; + }); + + const cachedOverview = await cachedPromise; + if (cachedOverview) { + return cachedOverview; + } + return await refreshPromise; } protected getRepositoryProvider(project: Project) { @@ -113,6 +129,10 @@ export class ProjectsService { } protected async onDidCreateProject(project: Project, installer: User) { + // Pre-fetch project details in the background -- don't await + this.getProjectOverviewCached(installer, project); + + // Install the prebuilds webhook if possible let { userId, teamId, cloneUrl } = project; const parsedUrl = RepoURL.parseRepoUrl(project.cloneUrl); const hostContext = parsedUrl?.host ? this.hostContextProvider.get(parsedUrl?.host) : undefined; diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index 3afaf2b1995ef4..9ef56575bf2baa 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -1644,7 +1644,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } await this.guardProjectOperation(user, projectId, "get"); try { - return await this.projectsService.getProjectOverview(user, project); + return await this.projectsService.getProjectOverviewCached(user, project); } catch (error) { if (UnauthorizedError.is(error)) { throw new ResponseError(ErrorCodes.NOT_AUTHENTICATED, "Unauthorized", error.data);