Skip to content

Commit 871f1c0

Browse files
zeripath6543
authored and
AbdulrhmnGhanem
committed
More efficiently parse shas for shaPostProcessor (go-gitea#16101)
* More efficiently parse shas for shaPostProcessor The shaPostProcessor currently repeatedly calls git rev-parse --verify on both backends which is fine if there is only one thing that matches a sha - however if there are multiple things then this becomes wildly inefficient. This PR provides functions for both backends which are much faster to use. Fix go-gitea#16092 * Add ShaExistCache to RenderContext Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de>
1 parent b26580f commit 871f1c0

11 files changed

+122
-10
lines changed

‎modules/git/repo_branch_gogit.go

+24
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,30 @@ import (
1313
"github.com/go-git/go-git/v5/plumbing"
1414
)
1515

16+
// IsObjectExist returns true if given reference exists in the repository.
17+
func (repo *Repository) IsObjectExist(name string) bool {
18+
if name == "" {
19+
return false
20+
}
21+
22+
_, err := repo.gogitRepo.ResolveRevision(plumbing.Revision(name))
23+
24+
return err == nil
25+
}
26+
27+
// IsReferenceExist returns true if given reference exists in the repository.
28+
func (repo *Repository) IsReferenceExist(name string) bool {
29+
if name == "" {
30+
return false
31+
}
32+
33+
reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
34+
if err != nil {
35+
return false
36+
}
37+
return reference.Type() != plumbing.InvalidReference
38+
}
39+
1640
// IsBranchExist returns true if given branch exists in current repository.
1741
func (repo *Repository) IsBranchExist(name string) bool {
1842
if name == "" {

‎modules/git/repo_branch_nogogit.go

+18
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,28 @@ package git
99

1010
import (
1111
"bufio"
12+
"bytes"
1213
"io"
1314
"strings"
1415
)
1516

17+
// IsObjectExist returns true if given reference exists in the repository.
18+
func (repo *Repository) IsObjectExist(name string) bool {
19+
if name == "" {
20+
return false
21+
}
22+
23+
wr, rd, cancel := repo.CatFileBatchCheck()
24+
defer cancel()
25+
_, err := wr.Write([]byte(name + "\n"))
26+
if err != nil {
27+
log("Error writing to CatFileBatchCheck %v", err)
28+
return false
29+
}
30+
sha, _, _, err := ReadBatchLine(rd)
31+
return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name)))
32+
}
33+
1634
// IsReferenceExist returns true if given reference exists in the repository.
1735
func (repo *Repository) IsReferenceExist(name string) bool {
1836
if name == "" {

‎modules/markup/html.go

+25-3
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM
286286
var nulCleaner = strings.NewReplacer("\000", "")
287287

288288
func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error {
289+
defer ctx.Cancel()
289290
// FIXME: don't read all content to memory
290291
rawHTML, err := ioutil.ReadAll(input)
291292
if err != nil {
@@ -996,6 +997,9 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
996997

997998
start := 0
998999
next := node.NextSibling
1000+
if ctx.ShaExistCache == nil {
1001+
ctx.ShaExistCache = make(map[string]bool)
1002+
}
9991003
for node != nil && node != next && start < len(node.Data) {
10001004
m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data[start:])
10011005
if m == nil {
@@ -1013,10 +1017,28 @@ func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
10131017
// as used by git and github for linking and thus we have to do similar.
10141018
// Because of this, we check to make sure that a matched hash is actually
10151019
// a commit in the repository before making it a link.
1016-
if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
1017-
if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
1018-
log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
1020+
1021+
// check cache first
1022+
exist, inCache := ctx.ShaExistCache[hash]
1023+
if !inCache {
1024+
if ctx.GitRepo == nil {
1025+
var err error
1026+
ctx.GitRepo, err = git.OpenRepository(ctx.Metas["repoPath"])
1027+
if err != nil {
1028+
log.Error("unable to open repository: %s Error: %v", ctx.Metas["repoPath"], err)
1029+
return
1030+
}
1031+
ctx.AddCancel(func() {
1032+
ctx.GitRepo.Close()
1033+
ctx.GitRepo = nil
1034+
})
10191035
}
1036+
1037+
exist = ctx.GitRepo.IsObjectExist(hash)
1038+
ctx.ShaExistCache[hash] = exist
1039+
}
1040+
1041+
if !exist {
10201042
start = m[3]
10211043
continue
10221044
}

‎modules/markup/renderer.go

+39-7
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"strings"
1414
"sync"
1515

16+
"code.gitea.io/gitea/modules/git"
1617
"code.gitea.io/gitea/modules/setting"
1718
)
1819

@@ -35,13 +36,44 @@ func Init() {
3536

3637
// RenderContext represents a render context
3738
type RenderContext struct {
38-
Ctx context.Context
39-
Filename string
40-
Type string
41-
IsWiki bool
42-
URLPrefix string
43-
Metas map[string]string
44-
DefaultLink string
39+
Ctx context.Context
40+
Filename string
41+
Type string
42+
IsWiki bool
43+
URLPrefix string
44+
Metas map[string]string
45+
DefaultLink string
46+
GitRepo *git.Repository
47+
ShaExistCache map[string]bool
48+
cancelFn func()
49+
}
50+
51+
// Cancel runs any cleanup functions that have been registered for this Ctx
52+
func (ctx *RenderContext) Cancel() {
53+
if ctx == nil {
54+
return
55+
}
56+
ctx.ShaExistCache = map[string]bool{}
57+
if ctx.cancelFn == nil {
58+
return
59+
}
60+
ctx.cancelFn()
61+
}
62+
63+
// AddCancel adds the provided fn as a Cleanup for this Ctx
64+
func (ctx *RenderContext) AddCancel(fn func()) {
65+
if ctx == nil {
66+
return
67+
}
68+
oldCancelFn := ctx.cancelFn
69+
if oldCancelFn == nil {
70+
ctx.cancelFn = fn
71+
return
72+
}
73+
ctx.cancelFn = func() {
74+
defer oldCancelFn()
75+
fn()
76+
}
4577
}
4678

4779
// Renderer defines an interface for rendering markup file to HTML

‎routers/web/org/home.go

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ func Home(ctx *context.Context) {
4141
desc, err := markdown.RenderString(&markup.RenderContext{
4242
URLPrefix: ctx.Repo.RepoLink,
4343
Metas: map[string]string{"mode": "document"},
44+
GitRepo: ctx.Repo.GitRepo,
4445
}, org.Description)
4546
if err != nil {
4647
ctx.ServerError("RenderString", err)

‎routers/web/repo/issue.go

+5
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,7 @@ func ViewIssue(ctx *context.Context) {
11371137
issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
11381138
URLPrefix: ctx.Repo.RepoLink,
11391139
Metas: ctx.Repo.Repository.ComposeMetas(),
1140+
GitRepo: ctx.Repo.GitRepo,
11401141
}, issue.Content)
11411142
if err != nil {
11421143
ctx.ServerError("RenderString", err)
@@ -1301,6 +1302,7 @@ func ViewIssue(ctx *context.Context) {
13011302
comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
13021303
URLPrefix: ctx.Repo.RepoLink,
13031304
Metas: ctx.Repo.Repository.ComposeMetas(),
1305+
GitRepo: ctx.Repo.GitRepo,
13041306
}, comment.Content)
13051307
if err != nil {
13061308
ctx.ServerError("RenderString", err)
@@ -1376,6 +1378,7 @@ func ViewIssue(ctx *context.Context) {
13761378
comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
13771379
URLPrefix: ctx.Repo.RepoLink,
13781380
Metas: ctx.Repo.Repository.ComposeMetas(),
1381+
GitRepo: ctx.Repo.GitRepo,
13791382
}, comment.Content)
13801383
if err != nil {
13811384
ctx.ServerError("RenderString", err)
@@ -1734,6 +1737,7 @@ func UpdateIssueContent(ctx *context.Context) {
17341737
content, err := markdown.RenderString(&markup.RenderContext{
17351738
URLPrefix: ctx.Query("context"),
17361739
Metas: ctx.Repo.Repository.ComposeMetas(),
1740+
GitRepo: ctx.Repo.GitRepo,
17371741
}, issue.Content)
17381742
if err != nil {
17391743
ctx.ServerError("RenderString", err)
@@ -2161,6 +2165,7 @@ func UpdateCommentContent(ctx *context.Context) {
21612165
content, err := markdown.RenderString(&markup.RenderContext{
21622166
URLPrefix: ctx.Query("context"),
21632167
Metas: ctx.Repo.Repository.ComposeMetas(),
2168+
GitRepo: ctx.Repo.GitRepo,
21642169
}, comment.Content)
21652170
if err != nil {
21662171
ctx.ServerError("RenderString", err)

‎routers/web/repo/milestone.go

+2
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ func Milestones(ctx *context.Context) {
8888
m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
8989
URLPrefix: ctx.Repo.RepoLink,
9090
Metas: ctx.Repo.Repository.ComposeMetas(),
91+
GitRepo: ctx.Repo.GitRepo,
9192
}, m.Content)
9293
if err != nil {
9394
ctx.ServerError("RenderString", err)
@@ -280,6 +281,7 @@ func MilestoneIssuesAndPulls(ctx *context.Context) {
280281
milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
281282
URLPrefix: ctx.Repo.RepoLink,
282283
Metas: ctx.Repo.Repository.ComposeMetas(),
284+
GitRepo: ctx.Repo.GitRepo,
283285
}, milestone.Content)
284286
if err != nil {
285287
ctx.ServerError("RenderString", err)

‎routers/web/repo/projects.go

+2
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ func Projects(ctx *context.Context) {
8181
projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{
8282
URLPrefix: ctx.Repo.RepoLink,
8383
Metas: ctx.Repo.Repository.ComposeMetas(),
84+
GitRepo: ctx.Repo.GitRepo,
8485
}, projects[i].Description)
8586
if err != nil {
8687
ctx.ServerError("RenderString", err)
@@ -322,6 +323,7 @@ func ViewProject(ctx *context.Context) {
322323
project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
323324
URLPrefix: ctx.Repo.RepoLink,
324325
Metas: ctx.Repo.Repository.ComposeMetas(),
326+
GitRepo: ctx.Repo.GitRepo,
325327
}, project.Description)
326328
if err != nil {
327329
ctx.ServerError("RenderString", err)

‎routers/web/repo/release.go

+2
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ func releasesOrTags(ctx *context.Context, isTagList bool) {
145145
r.Note, err = markdown.RenderString(&markup.RenderContext{
146146
URLPrefix: ctx.Repo.RepoLink,
147147
Metas: ctx.Repo.Repository.ComposeMetas(),
148+
GitRepo: ctx.Repo.GitRepo,
148149
}, r.Note)
149150
if err != nil {
150151
ctx.ServerError("RenderString", err)
@@ -213,6 +214,7 @@ func SingleRelease(ctx *context.Context) {
213214
release.Note, err = markdown.RenderString(&markup.RenderContext{
214215
URLPrefix: ctx.Repo.RepoLink,
215216
Metas: ctx.Repo.Repository.ComposeMetas(),
217+
GitRepo: ctx.Repo.GitRepo,
216218
}, release.Note)
217219
if err != nil {
218220
ctx.ServerError("RenderString", err)

‎routers/web/repo/view.go

+3
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
338338
Filename: readmeFile.name,
339339
URLPrefix: readmeTreelink,
340340
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
341+
GitRepo: ctx.Repo.GitRepo,
341342
}, rd, &result)
342343
if err != nil {
343344
log.Error("Render failed: %v then fallback", err)
@@ -512,6 +513,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
512513
Filename: blob.Name(),
513514
URLPrefix: path.Dir(treeLink),
514515
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
516+
GitRepo: ctx.Repo.GitRepo,
515517
}, rd, &result)
516518
if err != nil {
517519
ctx.ServerError("Render", err)
@@ -570,6 +572,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
570572
Filename: blob.Name(),
571573
URLPrefix: path.Dir(treeLink),
572574
Metas: ctx.Repo.Repository.ComposeDocumentMetas(),
575+
GitRepo: ctx.Repo.GitRepo,
573576
}, rd, &result)
574577
if err != nil {
575578
ctx.ServerError("Render", err)

‎routers/web/user/profile.go

+1
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ func Profile(ctx *context.Context) {
117117
content, err := markdown.RenderString(&markup.RenderContext{
118118
URLPrefix: ctx.Repo.RepoLink,
119119
Metas: map[string]string{"mode": "document"},
120+
GitRepo: ctx.Repo.GitRepo,
120121
}, ctxUser.Description)
121122
if err != nil {
122123
ctx.ServerError("RenderString", err)

0 commit comments

Comments
 (0)