Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[server] Improve logging / error messages of GitLab app and context parser #4747

Merged
merged 2 commits into from
Jul 13, 2021
Merged

[server] Improve logging / error messages of GitLab app and context parser #4747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e2439d8
[server] Improve logging / error messages of GitLab app and context p…
corneliusludmann Jul 8, 2021
36ff6c2
[server] Improve branch/tag API call in GitLab context parser
corneliusludmann Jul 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions components/server/ee/src/prebuilds/gitlab-app.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { StartPrebuildResult } from './github-app';
import { TokenService } from '../../../src/user/token-service';
import { HostContextProvider } from '../../../src/auth/host-context-provider';
import { GitlabService } from './gitlab-service';
import { log } from '@gitpod/gitpod-protocol/lib/util/logging';

@injectable()
export class GitLabApp {
Expand All @@ -29,19 +30,21 @@ export class GitLabApp {
@postConstruct()
protected init() {
this._router.post('/', async (req, res) => {
console.log(req.header('X-Gitlab-Event'));
if (req.header('X-Gitlab-Event') === 'Push Hook') {
const event = req.header('X-Gitlab-Event');
if (event === 'Push Hook') {
const context = req.body as GitLabPushHook;
const span = TraceContext.startSpan("GitLapApp.handleEvent", {});
span.setTag("request", context);
console.log(JSON.stringify(context, undefined, ' '));
log.debug("GitLab push hook received", { event, context });
const user = await this.findUser({span},context, req);
if (!user) {
res.statusCode = 503;
res.send();
return;
}
this.handlePushHook({span},context, user);
} else {
log.debug("Unknown GitLab event received", { event });
}
res.send('OK');
});
Expand All @@ -53,7 +56,7 @@ export class GitLabApp {
const secretToken = req.header('X-Gitlab-Token');
span.setTag('secret-token', secretToken);
if (!secretToken) {
throw new Error('No sceretToken provided.');
throw new Error('No secretToken provided.');
}
const [userid, tokenValue] = secretToken.split('|');
const user = await this.userDB.findUserById(userid);
Expand Down Expand Up @@ -85,14 +88,15 @@ export class GitLabApp {
const span = TraceContext.startSpan("GitLapApp.handlePushHook", ctx);
try {
const contextURL = this.createContextUrl(body);
span.setTag('contextURl', contextURL);
log.debug({ userId: user.id }, "GitLab push hook: Context URL", { context: body, contextURL });
span.setTag('contextURL', contextURL);
const config = await this.prebuildManager.fetchConfig({ span }, user, contextURL);
if (!this.prebuildManager.shouldPrebuild(config)) {
console.log('No config. No prebuild.');
log.debug({ userId: user.id }, "GitLab push hook: There is no prebuild config.", { context: body, contextURL });
return undefined;
}

console.log('Starting prebuild.', { contextURL, commit: body.after, gitUrl: body.repository.git_http_url })
log.debug({ userId: user.id }, "GitLab push hook: Starting prebuild", { context: body, contextURL });
const ws = await this.prebuildManager.startPrebuild({ span }, user, contextURL, body.repository.git_http_url, body.after);
return ws;
} finally {
Expand All @@ -102,9 +106,8 @@ export class GitLabApp {

protected createContextUrl(body: GitLabPushHook) {
const repoUrl = body.repository.git_http_url;
const contextUrl = `${repoUrl.substr(0, repoUrl.length - 4)}/tree${body.ref.substr('refs/head/'.length)}`;
console.log(contextUrl);
return contextUrl;
const contextURL = `${repoUrl.substr(0, repoUrl.length - 4)}/-/tree${body.ref.substr('refs/head/'.length)}`;
return contextURL;
}

get router(): express.Router {
Expand Down
6 changes: 6 additions & 0 deletions components/server/src/gitlab/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ export namespace GitLab {
export function is(something: any): something is ApiError {
return !!something && something.name === 'GitLabApiError';
}
export function isNotFound(error: ApiError): boolean {
return !!error.httpError?.description.startsWith("404");
}
export function isInternalServerError(error: ApiError): boolean {
return !!error.httpError?.description.startsWith("500");
}
}
/**
* https://github.com/gitlabhq/gitlabhq/blob/master/doc/api/projects.md#get-single-project
Expand Down
125 changes: 72 additions & 53 deletions components/server/src/gitlab/gitlab-context-parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,78 +170,97 @@ export class GitlabContextParser extends AbstractContextParser implements IConte
// https://gitlab.com/AlexTugarev/gp-test/blob/wip/folder/empty.file.jpeg
protected async handleTreeContext(user: User, host: string, owner: string, repoName: string, segments: string[]): Promise<NavigatorContext> {

const branchOrTagPromise = segments.length > 0 ? this.getBranchOrTag(user, owner, repoName, segments) : undefined;
const repository = await this.fetchRepo(user, `${owner}/${repoName}`);
const branchOrTag = await branchOrTagPromise;
const context = <NavigatorContext>{
isFile: false,
path: '',
title: `${owner}/${repoName}` + (branchOrTag ? ` - ${branchOrTag.name}` : ''),
ref: branchOrTag && branchOrTag.name,
revision: branchOrTag && branchOrTag.revision,
refType: branchOrTag && branchOrTag.type,
repository
};
if (!branchOrTag) {
return context;
}
if (segments.length === 1 || branchOrTag.fullPath.length === 0) {
return context;
}
try {
const branchOrTagPromise = segments.length > 0 ? this.getBranchOrTag(user, owner, repoName, segments) : undefined;
const repository = await this.fetchRepo(user, `${owner}/${repoName}`);
const branchOrTag = await branchOrTagPromise;
const context = <NavigatorContext>{
isFile: false,
path: '',
title: `${owner}/${repoName}` + (branchOrTag ? ` - ${branchOrTag.name}` : ''),
ref: branchOrTag && branchOrTag.name,
revision: branchOrTag && branchOrTag.revision,
refType: branchOrTag && branchOrTag.type,
repository
};
if (!branchOrTag) {
return context;
}
if (segments.length === 1 || branchOrTag.fullPath.length === 0) {
return context;
}

const result = await this.gitlabApi.run<GitLab.TreeObject[]>(user, async g => {
return g.Repositories.tree(`${owner}/${repoName}`, { ref: branchOrTag.name, path: path.dirname(branchOrTag.fullPath) });
});
if (GitLab.ApiError.is(result)) {
log.debug(`Error reading TREE ${owner}/${repoName}/tree/${segments.join('/')}.`, result);
} else {
const object = result.find(o => o.path === branchOrTag.fullPath);
if (object) {
const isFile = object.type === "blob";
context.isFile = isFile;
context.path = branchOrTag.fullPath;
const result = await this.gitlabApi.run<GitLab.TreeObject[]>(user, async g => {
return g.Repositories.tree(`${owner}/${repoName}`, { ref: branchOrTag.name, path: path.dirname(branchOrTag.fullPath) });
});
if (GitLab.ApiError.is(result)) {
throw new Error(`Error reading TREE ${owner}/${repoName}/tree/${segments.join('/')}: ${result}`);
} else {
const object = result.find(o => o.path === branchOrTag.fullPath);
if (object) {
const isFile = object.type === "blob";
context.isFile = isFile;
context.path = branchOrTag.fullPath;
}
}
return context;
} catch (e) {
log.debug("GitLab context parser: Error handle tree context.", e);
throw e;
}
return context;
}

protected async getBranchOrTag(user: User, owner: string, repoName: string, segments: string[]): Promise<{ type: RefType, name: string, revision: string, fullPath: string }> {

let branchOrTagObject: { type: RefType, name: string, revision: string } | undefined = undefined;

const search = `^${segments[0]}`; // search for tags/branches that start with the first segment
// `segments` could have branch/tag name parts as well as file path parts.
// We never know which segments belong to the branch/tag name and which are already folder names.
// Here we generate a list of candidates for branch/tag names.
const branchOrTagCandidates: string[] = [];
// Try the concatination of all segments first.
branchOrTagCandidates.push(segments.join("/"));
// Then all subsets.
for (let i = 1; i < segments.length; i++) {
branchOrTagCandidates.push(segments.slice(0, i).join("/"));
}

const possibleBranches = await this.gitlabApi.run<GitLab.Branch[]>(user, async g => {
return g.Branches.all(`${owner}/${repoName}`, { search });
});
if (!GitLab.ApiError.is(possibleBranches)) {
for (let candidate of possibleBranches) {
const name = candidate.name;
const nameSegements = candidate.name.split('/');
if (segments.length >= nameSegements.length && segments.slice(0, nameSegements.length).join('/') === name) {
branchOrTagObject = { type: 'branch', name, revision: candidate.commit.id };
break;
for (const candidate of branchOrTagCandidates) {

// Check if there is a BRANCH with name `candidate`:
const possibleBranch = await this.gitlabApi.run<GitLab.Branch>(user, async g => {
return g.Branches.show(`${owner}/${repoName}`, candidate);
});
// If the branch does not exist, the GitLab API returns with NotFound or InternalServerError.
const isNotFoundBranch = GitLab.ApiError.is(possibleBranch) && (GitLab.ApiError.isNotFound(possibleBranch) || GitLab.ApiError.isInternalServerError(possibleBranch));
if (!isNotFoundBranch) {
if (GitLab.ApiError.is(possibleBranch)) {
throw new Error(`GitLab ApiError on searching for possible branches for ${owner}/${repoName}/tree/${segments.join('/')}: ${possibleBranch}`);
}
branchOrTagObject = { type: 'branch', name: possibleBranch.name, revision: possibleBranch.commit.id };
break;
}
}

if (branchOrTagObject === undefined) {
const possibleTags = await this.gitlabApi.run<GitLab.Tag[]>(user, async g => {
return g.Tags.all(`${owner}/${repoName}`, { search });
// Check if there is a TAG with name `candidate`:
const possibleTag = await this.gitlabApi.run<GitLab.Tag>(user, async g => {
return g.Tags.show(`${owner}/${repoName}`, candidate);
});
if (!GitLab.ApiError.is(possibleTags)) {
for (let candidate of possibleTags) {
const name = candidate.name;
const nameSegements = candidate.name.split('/');
if (segments.length >= nameSegements.length && segments.slice(0, nameSegements.length).join('/') === name) {
branchOrTagObject = { type: 'tag', name, revision: candidate.commit.id };
break;
}
// If the tag does not exist, the GitLab API returns with NotFound or InternalServerError.
const isNotFoundTag = GitLab.ApiError.is(possibleTag) && (GitLab.ApiError.isNotFound(possibleTag) || GitLab.ApiError.isInternalServerError(possibleTag));
if (!isNotFoundTag) {
if (GitLab.ApiError.is(possibleTag)) {
throw new Error(`GitLab ApiError on searching for possible tags for ${owner}/${repoName}/tree/${segments.join('/')}: ${possibleTag}`);
}
branchOrTagObject = { type: 'tag', name: possibleTag.name, revision: possibleTag.commit.id };
break;
}
}

// There seems to be no matching branch or tag.
if (branchOrTagObject === undefined) {
log.debug(`Cannot find tag/branch for context: ${owner}/${repoName}/tree/${segments.join('/')}.`,
{ branchOrTagCandidates }
);
throw new Error(`Cannot find tag/branch for context: ${owner}/${repoName}/tree/${segments.join('/')}.`);
}

Expand Down