[server] Support Projects and Prebuilds with GitHub Enterprise reposi…

…tories

components/dashboard/src/projects/NewProject.tsx

@@ -525,17 +525,14 @@ function GitProviders(props: {
         });
     }
 
-    // for now we exclude GitHub Enterprise
-    const filteredProviders = () => props.authProviders.filter(p => p.host === "github.com" || p.host === "bitbucket.org" || p.authProviderType === "GitLab");
-
     return (
         <div className="mt-8 border rounded-t-xl border-gray-100 dark:border-gray-800 flex-col">
             <div className="p-6 p-b-0">
                 <div className="text-center text-gray-500">
                     Select a Git provider first and continue with your repositories.
                 </div>
                 <div className="mt-6 flex flex-col space-y-3 items-center pb-8">
-                    {filteredProviders().map(ap => {
+                    {props.authProviders.map(ap => {
                         return (
                             <button key={"button" + ap.host} className="btn-login flex-none w-56 h-10 p-0 inline-flex" onClick={() => selectProvider(ap)}>
                                 {iconForAuthProvider(ap.authProviderType)}

components/server/ee/src/auth/host-container-mapping.ts

@@ -8,6 +8,7 @@ import { injectable, interfaces } from "inversify";
 import { HostContainerMapping } from "../../../src/auth/host-container-mapping";
 import { gitlabContainerModuleEE } from "../gitlab/container-module";
 import { bitbucketContainerModuleEE } from "../bitbucket/container-module";
+import { gitHubContainerModuleEE } from "../github/container-module";
 
 @injectable()
 export class HostContainerMappingEE extends HostContainerMapping {
@@ -23,6 +24,8 @@ export class HostContainerMappingEE extends HostContainerMapping {
         // case "BitbucketServer":
             // FIXME
             // return (modules || []).concat([bitbucketContainerModuleEE]);
+        case "GitHub":
+            return (modules || []).concat([gitHubContainerModuleEE]);
         default:
             return modules;
         }

components/server/ee/src/container-module.ts

@@ -23,6 +23,7 @@ import { GithubAppRules } from "./prebuilds/github-app-rules";
 import { PrebuildStatusMaintainer } from "./prebuilds/prebuilt-status-maintainer";
 import { GitLabApp } from "./prebuilds/gitlab-app";
 import { BitbucketApp } from "./prebuilds/bitbucket-app";
+import { GitHubEnterpriseApp } from "./prebuilds/github-enterprise-app";
 import { IPrefixContextParser } from "../../src/workspace/context-parser";
 import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser";
 import { StartIncrementalPrebuildContextParser } from "./prebuilds/start-incremental-prebuild-context-parser";
@@ -68,6 +69,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
     bind(GitLabAppSupport).toSelf().inSingletonScope();
     bind(BitbucketApp).toSelf().inSingletonScope();
     bind(BitbucketAppSupport).toSelf().inSingletonScope();
+    bind(GitHubEnterpriseApp).toSelf().inSingletonScope();
 
     bind(LicenseEvaluator).toSelf().inSingletonScope();
     bind(LicenseKeySource).to(DBLicenseKeySource).inSingletonScope();

components/server/ee/src/github/container-module.ts

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the Gitpod Enterprise Source Code License,
+ * See License.enterprise.txt in the project root folder.
+ */
+
+import { ContainerModule } from "inversify";
+import { RepositoryService } from "../../../src/repohost/repo-service";
+import { GitHubService } from "../prebuilds/github-service";
+
+export const gitHubContainerModuleEE = new ContainerModule((_bind, _unbind, _isBound, rebind) => {
+    rebind(RepositoryService).to(GitHubService).inSingletonScope();
+});

components/server/ee/src/prebuilds/github-enterprise-app.ts

@@ -0,0 +1,197 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the Gitpod Enterprise Source Code License,
+ * See License.enterprise.txt in the project root folder.
+ */
+
+import * as express from 'express';
+import { createHmac } from 'crypto';
+import { postConstruct, injectable, inject } from 'inversify';
+import { ProjectDB, TeamDB, UserDB } from '@gitpod/gitpod-db/lib';
+import { PrebuildManager } from '../prebuilds/prebuild-manager';
+import { TraceContext } from '@gitpod/gitpod-protocol/lib/util/tracing';
+import { TokenService } from '../../../src/user/token-service';
+import { HostContextProvider } from '../../../src/auth/host-context-provider';
+import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
+import { Project, StartPrebuildResult, User } from '@gitpod/gitpod-protocol';
+import { GitHubService } from './github-service';
+import { URL } from 'url';
+
+@injectable()
+export class GitHubEnterpriseApp {
+
+    @inject(UserDB) protected readonly userDB: UserDB;
+    @inject(PrebuildManager) protected readonly prebuildManager: PrebuildManager;
+    @inject(TokenService) protected readonly tokenService: TokenService;
+    @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
+    @inject(ProjectDB) protected readonly projectDB: ProjectDB;
+    @inject(TeamDB) protected readonly teamDB: TeamDB;
+
+    protected _router = express.Router();
+    public static path = '/apps/ghe/';
+
+    @postConstruct()
+    protected init() {
+        this._router.post('/', async (req, res) => {
+            const event = req.header('X-Github-Event');
+            if (event === 'push') {
+                const payload = req.body as GitHubEnterprisePushPayload;
+                const span = TraceContext.startSpan("GitHubEnterpriseApp.handleEvent", {});
+                span.setTag("payload", payload);
+                let user: User | undefined;
+                try {
+                    user = await this.findUser({ span }, payload, req);
+                } catch (error) {
+                    log.error("Cannot find user.", error, { req })
+                }
+                if (!user) {
+                    res.statusCode = 401;
+                    res.send();
+                    return;
+                }
+                await this.handlePushHook({ span }, payload, user);
+            } else {
+                log.info("Unknown GitHub Enterprise event received", { event });
+            }
+            res.send('OK');
+        });
+    }
+
+    protected async findUser(ctx: TraceContext, payload: GitHubEnterprisePushPayload, req: express.Request): Promise<User> {
+        const span = TraceContext.startSpan("GitHubEnterpriseApp.findUser", ctx);
+        try {
+            const host = req.header('X-Github-Enterprise-Host');
+            const hostContext = this.hostContextProvider.get(host || '');
+            if (!host || !hostContext) {
+                throw new Error('Unsupported GitHub Enterprise host: ' + host);
+            }
+            const { authProviderId } = hostContext.authProvider;
+            const authId = payload.sender.id;
+            const user = await this.userDB.findUserByIdentity({ authProviderId, authId });
+            if (!user) {
+                throw new Error(`No user found with identity ${authProviderId}/${authId}.`);
+            } else if (!!user.blocked) {
+                throw new Error(`Blocked user ${user.id} tried to start prebuild.`);
+            }
+            const gitpodIdentity = user.identities.find(i => i.authProviderId === TokenService.GITPOD_AUTH_PROVIDER_ID);
+            if (!gitpodIdentity) {
+                throw new Error(`User ${user.id} has no identity for '${TokenService.GITPOD_AUTH_PROVIDER_ID}'.`);
+            }
+            // Verify the webhook signature
+            const signature = req.header('X-Hub-Signature-256');
+            const body = (req as any).rawBody;
+            const tokenEntries = (await this.userDB.findTokensForIdentity(gitpodIdentity)).filter(tokenEntry => {
+                return tokenEntry.token.scopes.includes(GitHubService.PREBUILD_TOKEN_SCOPE);
+            });
+            const signingToken = tokenEntries.find(tokenEntry => {
+                const sig = 'sha256=' + createHmac('sha256', user.id + '|' + tokenEntry.token.value)
+                    .update(body)
+                    .digest('hex');
+                return sig === signature;
+            });
+            if (!signingToken) {
+                throw new Error(`User ${user.id} has no token matching the payload signature.`);
+            }
+            return user;
+        } finally {
+            span.finish();
+        }
+    }
+
+    protected async handlePushHook(ctx: TraceContext, payload: GitHubEnterprisePushPayload, user: User): Promise<StartPrebuildResult | undefined> {
+        const span = TraceContext.startSpan("GitHubEnterpriseApp.handlePushHook", ctx);
+        try {
+            const contextURL = this.createContextUrl(payload);
+            span.setTag('contextURL', contextURL);
+            const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
+            if (!this.prebuildManager.shouldPrebuild(config)) {
+                log.info('GitHub Enterprise push event: No config. No prebuild.');
+                return undefined;
+            }
+
+            log.debug('GitHub Enterprise push event: Starting prebuild.', { contextURL });
+
+            const cloneURL = payload.repository.clone_url;
+            const projectAndOwner = await this.findProjectAndOwner(cloneURL, user);
+
+            const ws = await this.prebuildManager.startPrebuild({ span }, {
+                user: projectAndOwner.user,
+                project: projectAndOwner?.project,
+                branch: this.getBranchFromRef(payload.ref),
+                contextURL,
+                cloneURL,
+                commit: payload.after,
+            });
+            return ws;
+        } finally {
+            span.finish();
+        }
+    }
+
+    /**
+     * Finds the relevant user account and project to the provided webhook event information.
+     *
+     * First of all it tries to find the project for the given `cloneURL`, then it tries to
+     * find the installer, which is also supposed to be a team member. As a fallback, it
+     * looks for a team member which also has a connection with this GitHub Enterprise server.
+     *
+     * @param cloneURL of the webhook event
+     * @param webhookInstaller the user account known from the webhook installation
+     * @returns a promise which resolves to a user account and an optional project.
+     */
+    protected async findProjectAndOwner(cloneURL: string, webhookInstaller: User): Promise<{ user: User, project?: Project }> {
+        const project = await this.projectDB.findProjectByCloneUrl(cloneURL);
+        if (project) {
+            if (project.userId) {
+                const user = await this.userDB.findUserById(project.userId);
+                if (user) {
+                    return { user, project };
+                }
+            } else if (project.teamId) {
+                const teamMembers = await this.teamDB.findMembersByTeam(project.teamId || '');
+                if (teamMembers.some(t => t.userId === webhookInstaller.id)) {
+                    return { user: webhookInstaller, project };
+                }
+                const hostContext = this.hostContextProvider.get(new URL(cloneURL).host);
+                const authProviderId = hostContext?.authProvider.authProviderId;
+                for (const teamMember of teamMembers) {
+                    const user = await this.userDB.findUserById(teamMember.userId);
+                    if (user && user.identities.some(i => i.authProviderId === authProviderId)) {
+                        return { user, project };
+                    }
+                }
+            }
+        }
+        return { user: webhookInstaller };
+    }
+
+    protected getBranchFromRef(ref: string): string | undefined {
+        const headsPrefix = "refs/heads/";
+        if (ref.startsWith(headsPrefix)) {
+            return ref.substring(headsPrefix.length);
+        }
+
+        return undefined;
+    }
+
+    protected createContextUrl(payload: GitHubEnterprisePushPayload) {
+        return `${payload.repository.url}/tree/${this.getBranchFromRef(payload.ref)}`;
+    }
+
+    get router(): express.Router {
+        return this._router;
+    }
+}
+
+interface GitHubEnterprisePushPayload {
+    ref: string;
+    after: string;
+    repository: {
+        url: string;
+        clone_url: string;
+    };
+    sender: {
+        login: string;
+        id: string;
+    };
+}

components/server/ee/src/prebuilds/github-service.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the Gitpod Enterprise Source Code License,
+ * See License.enterprise.txt in the project root folder.
+ */
+
+import { RepositoryService } from "../../../src/repohost/repo-service";
+import { inject, injectable } from "inversify";
+import { GitHubGraphQlEndpoint, GitHubRestApi } from "../../../src/github/api";
+import { GitHubEnterpriseApp } from "./github-enterprise-app";
+import { AuthProviderParams } from "../../../src/auth/auth-provider";
+import { GithubContextParser } from "../../../src/github/github-context-parser";
+import { User } from "@gitpod/gitpod-protocol";
+import { Config } from "../../../src/config";
+import { TokenService } from "../../../src/user/token-service";
+
+@injectable()
+export class GitHubService extends RepositoryService {
+
+    static PREBUILD_TOKEN_SCOPE = 'prebuilds';
+
+    @inject(GitHubGraphQlEndpoint) protected readonly githubQueryApi: GitHubGraphQlEndpoint;
+    @inject(GitHubRestApi) protected readonly githubApi: GitHubRestApi;
+    @inject(Config) protected readonly config: Config;
+    @inject(AuthProviderParams) protected authProviderConfig: AuthProviderParams;
+    @inject(TokenService) protected tokenService: TokenService;
+    @inject(GithubContextParser) protected githubContextParser: GithubContextParser;
+
+    async canInstallAutomatedPrebuilds(user: User, cloneUrl: string): Promise<boolean> {
+        const { host } = await this.githubContextParser.parseURL(user, cloneUrl);
+        if (host === 'github.com' || host !== this.authProviderConfig.host) {
+            return false;
+        }
+        return true;
+    }
+
+    async installAutomatedPrebuilds(user: User, cloneUrl: string): Promise<void> {
+        const { owner, repoName: repo } = await this.githubContextParser.parseURL(user, cloneUrl);
+        const webhooks = (await this.githubApi.run(user, gh => gh.repos.listWebhooks({ owner, repo }))).data;
+        for (const webhook of webhooks) {
+            if (webhook.config.url === this.getHookUrl()) {
+                await this.githubApi.run(user, gh => gh.repos.deleteWebhook({ owner, repo, hook_id: webhook.id }));
+            }
+        }
+        // TODO(janx): Also delete old tokens with scopes `GitHubService.PREBUILD_TOKEN_SCOPE` and `cloneUrl`?
+        const tokenEntry = await this.tokenService.createGitpodToken(user, GitHubService.PREBUILD_TOKEN_SCOPE, cloneUrl);
+        const config = {
+            url: this.getHookUrl(),
+            content_type: 'json',
+            secret: user.id + '|' + tokenEntry.token.value,
+        };
+        await this.githubApi.run(user, gh => gh.repos.createWebhook({ owner, repo, config }));
+    }
+
+    protected getHookUrl() {
+        return this.config.hostUrl.with({
+            pathname: GitHubEnterpriseApp.path
+        }).toString();
+    }
+}

components/server/ee/src/server.ts

@@ -13,12 +13,14 @@ import { GitLabApp } from './prebuilds/gitlab-app';
 import { BitbucketApp } from './prebuilds/bitbucket-app';
 import { GithubApp } from './prebuilds/github-app';
 import { SnapshotService } from './workspace/snapshot-service';
+import { GitHubEnterpriseApp } from './prebuilds/github-enterprise-app';
 
 export class ServerEE<C extends GitpodClient, S extends GitpodServer> extends Server<C, S> {
     @inject(GithubApp) protected readonly githubApp: GithubApp;
     @inject(GitLabApp) protected readonly gitLabApp: GitLabApp;
     @inject(BitbucketApp) protected readonly bitbucketApp: BitbucketApp;
     @inject(SnapshotService) protected readonly snapshotService: SnapshotService;
+    @inject(GitHubEnterpriseApp) protected readonly gitHubEnterpriseApp: GitHubEnterpriseApp;
 
     public async init(app: express.Application) {
         await super.init(app);
@@ -43,5 +45,8 @@ export class ServerEE<C extends GitpodClient, S extends GitpodServer> extends Se
 
         log.info("Registered Bitbucket app at " + BitbucketApp.path);
         app.use(BitbucketApp.path, this.bitbucketApp.router);
+
+        log.info("Registered GitHub EnterpriseApp app at " + GitHubEnterpriseApp.path);
+        app.use(GitHubEnterpriseApp.path, this.gitHubEnterpriseApp.router);
     }
 }

components/server/ee/src/workspace/gitpod-server-impl.ts

@@ -1476,6 +1476,11 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
 
         if (providerHost === "github.com") {
             repositories.push(...(await this.githubAppSupport.getProviderRepositoriesForUser({ user, ...params })));
+        } else if (provider?.authProviderType === "GitHub") {
+            const hostContext = this.hostContextProvider.get(providerHost);
+            if (hostContext?.services) {
+                repositories.push(...(await hostContext.services.repositoryProvider.getProviderRepos(user)));
+            }
         } else if (providerHost === "bitbucket.org" && provider) {
             repositories.push(...(await this.bitbucketAppSupport.getProviderRepositoriesForUser({ user, provider })));
         } else if (provider?.authProviderType === "GitLab") {

components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts

@@ -4,7 +4,7 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-import { Branch, CommitInfo, Repository, User } from "@gitpod/gitpod-protocol";
+import { Branch, CommitInfo, ProviderRepository, Repository, User } from "@gitpod/gitpod-protocol";
 import { injectable } from 'inversify';
 import { RepositoryProvider } from '../repohost/repository-provider';
 
@@ -102,6 +102,11 @@ export class BitbucketServerRepositoryProvider implements RepositoryProvider {
         return [];
     }
 
+    async getProviderRepos(user: User): Promise<ProviderRepository[]> {
+        // FIXME(janx): Not implemented yet
+        return [];
+    }
+
     async hasReadAccess(user: User, owner: string, repo: string): Promise<boolean> {
         // TODO(janx): Not implemented yet
         return false;