From 2b422100a15ff4992f41317e51d3eefb68e7edee Mon Sep 17 00:00:00 2001 From: Gennady Kovshenin Date: Wed, 5 Oct 2022 02:43:43 +0300 Subject: [PATCH] Add `stat` to `ToCommit` function for speed Calls to ToCommit are very slow due to fetching diffs, analyzing files. This patch lets us supply `stat` as false to speed fetching a commit when we don't need the diff. /v1/repo/commits has a default `stat` set as true now. Set to false to experience fetching thousands of commits per second instead of 2-5 per second. --- modules/convert/git_commit.go | 59 ++++++++++++++++++---------------- routers/api/v1/repo/commits.go | 11 +++++-- routers/api/v1/repo/notes.go | 2 +- routers/api/v1/repo/pull.go | 2 +- templates/swagger/v1_json.tmpl | 6 ++++ 5 files changed, 49 insertions(+), 31 deletions(-) diff --git a/modules/convert/git_commit.go b/modules/convert/git_commit.go index dfd6cb080ca35..6015a73712896 100644 --- a/modules/convert/git_commit.go +++ b/modules/convert/git_commit.go @@ -73,7 +73,7 @@ func ToPayloadCommit(repo *repo_model.Repository, c *git.Commit) *api.PayloadCom } // ToCommit convert a git.Commit to api.Commit -func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit, userCache map[string]*user_model.User) (*api.Commit, error) { +func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git.Commit, userCache map[string]*user_model.User, stat bool) (*api.Commit, error) { var apiAuthor, apiCommitter *api.User // Retrieve author and committer information @@ -133,28 +133,7 @@ func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git. } } - // Retrieve files affected by the commit - fileStatus, err := git.GetCommitFileStatus(gitRepo.Ctx, repo.RepoPath(), commit.ID.String()) - if err != nil { - return nil, err - } - affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatus.Added)+len(fileStatus.Removed)+len(fileStatus.Modified)) - for _, files := range [][]string{fileStatus.Added, fileStatus.Removed, fileStatus.Modified} { - for _, filename := range files { - affectedFileList = append(affectedFileList, &api.CommitAffectedFiles{ - Filename: filename, - }) - } - } - - diff, err := gitdiff.GetDiff(gitRepo, &gitdiff.DiffOptions{ - AfterCommitID: commit.ID.String(), - }) - if err != nil { - return nil, err - } - - return &api.Commit{ + res := &api.Commit{ CommitMeta: &api.CommitMeta{ URL: repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String()), SHA: commit.ID.String(), @@ -188,11 +167,37 @@ func ToCommit(repo *repo_model.Repository, gitRepo *git.Repository, commit *git. Author: apiAuthor, Committer: apiCommitter, Parents: apiParents, - Files: affectedFileList, - Stats: &api.CommitStats{ + } + + // Retrieve files affected by the commit + if stat { + fileStatus, err := git.GetCommitFileStatus(gitRepo.Ctx, repo.RepoPath(), commit.ID.String()) + if err != nil { + return nil, err + } + affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatus.Added)+len(fileStatus.Removed)+len(fileStatus.Modified)) + for _, files := range [][]string{fileStatus.Added, fileStatus.Removed, fileStatus.Modified} { + for _, filename := range files { + affectedFileList = append(affectedFileList, &api.CommitAffectedFiles{ + Filename: filename, + }) + } + } + + diff, err := gitdiff.GetDiff(gitRepo, &gitdiff.DiffOptions{ + AfterCommitID: commit.ID.String(), + }) + if err != nil { + return nil, err + } + + res.Files = affectedFileList + res.Stats = &api.CommitStats{ Total: diff.TotalAddition + diff.TotalDeletion, Additions: diff.TotalAddition, Deletions: diff.TotalDeletion, - }, - }, nil + } + } + + return res, nil } diff --git a/routers/api/v1/repo/commits.go b/routers/api/v1/repo/commits.go index 12c329c2030bb..4e77c3f2f5dfa 100644 --- a/routers/api/v1/repo/commits.go +++ b/routers/api/v1/repo/commits.go @@ -70,7 +70,7 @@ func getCommit(ctx *context.APIContext, identifier string) { return } - json, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, nil) + json, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, nil, true) if err != nil { ctx.Error(http.StatusInternalServerError, "toCommit", err) return @@ -104,6 +104,10 @@ func GetAllCommits(ctx *context.APIContext) { // in: query // description: filepath of a file/dir // type: string + // - name: stat + // in: query + // description: include diff stats for every commit (disable for speedup, default 'true') + // type: boolean // - name: page // in: query // description: page number of results to return (1-based) @@ -209,9 +213,12 @@ func GetAllCommits(ctx *context.APIContext) { userCache := make(map[string]*user_model.User) apiCommits := make([]*api.Commit, len(commits)) + + stat := ctx.FormString("stat") == "" || ctx.FormBool("stat") + for i, commit := range commits { // Create json struct - apiCommits[i], err = convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, userCache) + apiCommits[i], err = convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, commit, userCache, stat) if err != nil { ctx.Error(http.StatusInternalServerError, "toCommit", err) return diff --git a/routers/api/v1/repo/notes.go b/routers/api/v1/repo/notes.go index 67f097a424aca..ee3133adeceef 100644 --- a/routers/api/v1/repo/notes.go +++ b/routers/api/v1/repo/notes.go @@ -69,7 +69,7 @@ func getNote(ctx *context.APIContext, identifier string) { return } - cmt, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, note.Commit, nil) + cmt, err := convert.ToCommit(ctx.Repo.Repository, ctx.Repo.GitRepo, note.Commit, nil, true) if err != nil { ctx.Error(http.StatusInternalServerError, "ToCommit", err) return diff --git a/routers/api/v1/repo/pull.go b/routers/api/v1/repo/pull.go index 2cf30e7c47b82..f6507dceba89a 100644 --- a/routers/api/v1/repo/pull.go +++ b/routers/api/v1/repo/pull.go @@ -1306,7 +1306,7 @@ func GetPullRequestCommits(ctx *context.APIContext) { apiCommits := make([]*api.Commit, 0, end-start) for i := start; i < end; i++ { - apiCommit, err := convert.ToCommit(ctx.Repo.Repository, baseGitRepo, commits[i], userCache) + apiCommit, err := convert.ToCommit(ctx.Repo.Repository, baseGitRepo, commits[i], userCache, true) if err != nil { ctx.ServerError("toCommit", err) return diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index b0e94b8bb5b33..99b85a2458016 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -3264,6 +3264,12 @@ "name": "path", "in": "query" }, + { + "type": "boolean", + "description": "include diff stats for every commit (disable for speedup, default 'true')", + "name": "stat", + "in": "query" + }, { "type": "integer", "description": "page number of results to return (1-based)",