Skip to content

Commit 1162cbc

Browse files
GiteaBotlunny
GiteaBot
and
lunny
authored
Performance optimization for tags synchronization (#34355) (#34522)
Backport #34355 by @lunny The tags synchronization is very slow for a non-mirror repository with many tags especially forking. This PR make all repositories' tags synchronization use the same function and remove the low performance synchronization function. The commit count of tag now will not be stored into database when syncing. Since the commits count will always be read from cache or git data, the `NumCommits` in the release table will be updated for the first read from git data. Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
1 parent 038990e commit 1162cbc

File tree

7 files changed

+31
-190
lines changed

7 files changed

+31
-190
lines changed

‎cmd/hook.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
)
2525

2626
const (
27-
hookBatchSize = 30
27+
hookBatchSize = 500
2828
)
2929

3030
var (

‎models/repo/release.go‎

Lines changed: 7 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,11 @@ func UpdateRelease(ctx context.Context, rel *Release) error {
161161
return err
162162
}
163163

164+
func UpdateReleaseNumCommits(ctx context.Context, rel *Release) error {
165+
_, err := db.GetEngine(ctx).ID(rel.ID).Cols("num_commits").Update(rel)
166+
return err
167+
}
168+
164169
// AddReleaseAttachments adds a release attachments
165170
func AddReleaseAttachments(ctx context.Context, releaseID int64, attachmentUUIDs []string) (err error) {
166171
// Check attachments
@@ -418,8 +423,8 @@ func UpdateReleasesMigrationsByType(ctx context.Context, gitServiceType structs.
418423
return err
419424
}
420425

421-
// PushUpdateDeleteTagsContext updates a number of delete tags with context
422-
func PushUpdateDeleteTagsContext(ctx context.Context, repo *Repository, tags []string) error {
426+
// PushUpdateDeleteTags updates a number of delete tags with context
427+
func PushUpdateDeleteTags(ctx context.Context, repo *Repository, tags []string) error {
423428
if len(tags) == 0 {
424429
return nil
425430
}
@@ -448,58 +453,6 @@ func PushUpdateDeleteTagsContext(ctx context.Context, repo *Repository, tags []s
448453
return nil
449454
}
450455

451-
// PushUpdateDeleteTag must be called for any push actions to delete tag
452-
func PushUpdateDeleteTag(ctx context.Context, repo *Repository, tagName string) error {
453-
rel, err := GetRelease(ctx, repo.ID, tagName)
454-
if err != nil {
455-
if IsErrReleaseNotExist(err) {
456-
return nil
457-
}
458-
return fmt.Errorf("GetRelease: %w", err)
459-
}
460-
if rel.IsTag {
461-
if _, err = db.DeleteByID[Release](ctx, rel.ID); err != nil {
462-
return fmt.Errorf("Delete: %w", err)
463-
}
464-
} else {
465-
rel.IsDraft = true
466-
rel.NumCommits = 0
467-
rel.Sha1 = ""
468-
if _, err = db.GetEngine(ctx).ID(rel.ID).AllCols().Update(rel); err != nil {
469-
return fmt.Errorf("Update: %w", err)
470-
}
471-
}
472-
473-
return nil
474-
}
475-
476-
// SaveOrUpdateTag must be called for any push actions to add tag
477-
func SaveOrUpdateTag(ctx context.Context, repo *Repository, newRel *Release) error {
478-
rel, err := GetRelease(ctx, repo.ID, newRel.TagName)
479-
if err != nil && !IsErrReleaseNotExist(err) {
480-
return fmt.Errorf("GetRelease: %w", err)
481-
}
482-
483-
if rel == nil {
484-
rel = newRel
485-
if _, err = db.GetEngine(ctx).Insert(rel); err != nil {
486-
return fmt.Errorf("InsertOne: %w", err)
487-
}
488-
} else {
489-
rel.Sha1 = newRel.Sha1
490-
rel.CreatedUnix = newRel.CreatedUnix
491-
rel.NumCommits = newRel.NumCommits
492-
rel.IsDraft = false
493-
if rel.IsTag && newRel.PublisherID > 0 {
494-
rel.PublisherID = newRel.PublisherID
495-
}
496-
if _, err = db.GetEngine(ctx).ID(rel.ID).AllCols().Update(rel); err != nil {
497-
return fmt.Errorf("Update: %w", err)
498-
}
499-
}
500-
return nil
501-
}
502-
503456
// RemapExternalUser ExternalUserRemappable interface
504457
func (r *Release) RemapExternalUser(externalName string, externalID, userID int64) error {
505458
r.OriginalAuthor = externalName

‎modules/repository/repo.go‎

Lines changed: 9 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,10 @@ import (
99
"fmt"
1010
"io"
1111
"strings"
12-
"time"
1312

1413
"code.gitea.io/gitea/models/db"
1514
git_model "code.gitea.io/gitea/models/git"
1615
repo_model "code.gitea.io/gitea/models/repo"
17-
user_model "code.gitea.io/gitea/models/user"
18-
"code.gitea.io/gitea/modules/container"
1916
"code.gitea.io/gitea/modules/git"
2017
"code.gitea.io/gitea/modules/gitrepo"
2118
"code.gitea.io/gitea/modules/lfs"
@@ -59,118 +56,6 @@ func SyncRepoTags(ctx context.Context, repoID int64) error {
5956
return SyncReleasesWithTags(ctx, repo, gitRepo)
6057
}
6158

62-
// SyncReleasesWithTags synchronizes release table with repository tags
63-
func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
64-
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
65-
66-
// optimized procedure for pull-mirrors which saves a lot of time (in
67-
// particular for repos with many tags).
68-
if repo.IsMirror {
69-
return pullMirrorReleaseSync(ctx, repo, gitRepo)
70-
}
71-
72-
existingRelTags := make(container.Set[string])
73-
opts := repo_model.FindReleasesOptions{
74-
IncludeDrafts: true,
75-
IncludeTags: true,
76-
ListOptions: db.ListOptions{PageSize: 50},
77-
RepoID: repo.ID,
78-
}
79-
for page := 1; ; page++ {
80-
opts.Page = page
81-
rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts)
82-
if err != nil {
83-
return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
84-
}
85-
if len(rels) == 0 {
86-
break
87-
}
88-
for _, rel := range rels {
89-
if rel.IsDraft {
90-
continue
91-
}
92-
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
93-
if err != nil && !git.IsErrNotExist(err) {
94-
return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err)
95-
}
96-
if git.IsErrNotExist(err) || commitID != rel.Sha1 {
97-
if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil {
98-
return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err)
99-
}
100-
} else {
101-
existingRelTags.Add(strings.ToLower(rel.TagName))
102-
}
103-
}
104-
}
105-
106-
_, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error {
107-
tagName := strings.TrimPrefix(refname, git.TagPrefix)
108-
if existingRelTags.Contains(strings.ToLower(tagName)) {
109-
return nil
110-
}
111-
112-
if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil {
113-
// sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11
114-
// this is a tree object, not a tag object which created before git
115-
log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err)
116-
}
117-
118-
return nil
119-
})
120-
return err
121-
}
122-
123-
// PushUpdateAddTag must be called for any push actions to add tag
124-
func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error {
125-
tag, err := gitRepo.GetTagWithID(sha1, tagName)
126-
if err != nil {
127-
return fmt.Errorf("unable to GetTag: %w", err)
128-
}
129-
commit, err := gitRepo.GetTagCommit(tag.Name)
130-
if err != nil {
131-
return fmt.Errorf("unable to get tag Commit: %w", err)
132-
}
133-
134-
sig := tag.Tagger
135-
if sig == nil {
136-
sig = commit.Author
137-
}
138-
if sig == nil {
139-
sig = commit.Committer
140-
}
141-
142-
var author *user_model.User
143-
createdAt := time.Unix(1, 0)
144-
145-
if sig != nil {
146-
author, err = user_model.GetUserByEmail(ctx, sig.Email)
147-
if err != nil && !user_model.IsErrUserNotExist(err) {
148-
return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err)
149-
}
150-
createdAt = sig.When
151-
}
152-
153-
commitsCount, err := commit.CommitsCount()
154-
if err != nil {
155-
return fmt.Errorf("unable to get CommitsCount: %w", err)
156-
}
157-
158-
rel := repo_model.Release{
159-
RepoID: repo.ID,
160-
TagName: tagName,
161-
LowerTagName: strings.ToLower(tagName),
162-
Sha1: commit.ID.String(),
163-
NumCommits: commitsCount,
164-
CreatedUnix: timeutil.TimeStamp(createdAt.Unix()),
165-
IsTag: true,
166-
}
167-
if author != nil {
168-
rel.PublisherID = author.ID
169-
}
170-
171-
return repo_model.SaveOrUpdateTag(ctx, repo, &rel)
172-
}
173-
17459
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
17560
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error {
17661
contentStore := lfs.NewContentStore()
@@ -286,18 +171,19 @@ func (shortRelease) TableName() string {
286171
return "release"
287172
}
288173

289-
// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
174+
// SyncReleasesWithTags is a tag<->release table
290175
// synchronization which overwrites all Releases from the repository tags. This
291176
// can be relied on since a pull-mirror is always identical to its
292-
// upstream. Hence, after each sync we want the pull-mirror release set to be
177+
// upstream. Hence, after each sync we want the release set to be
293178
// identical to the upstream tag set. This is much more efficient for
294179
// repositories like https://github.com/vim/vim (with over 13000 tags).
295-
func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
296-
log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
297-
tags, numTags, err := gitRepo.GetTagInfos(0, 0)
180+
func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
181+
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
182+
tags, _, err := gitRepo.GetTagInfos(0, 0)
298183
if err != nil {
299184
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
300185
}
186+
var added, deleted, updated int
301187
err = db.WithTx(ctx, func(ctx context.Context) error {
302188
dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{
303189
RepoID: repo.ID,
@@ -318,9 +204,7 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
318204
TagName: tag.Name,
319205
LowerTagName: strings.ToLower(tag.Name),
320206
Sha1: tag.Object.String(),
321-
// NOTE: ignored, since NumCommits are unused
322-
// for pull-mirrors (only relevant when
323-
// displaying releases, IsTag: false)
207+
// NOTE: ignored, The NumCommits value is calculated and cached on demand when the UI requires it.
324208
NumCommits: -1,
325209
CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
326210
IsTag: true,
@@ -349,13 +233,14 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
349233
return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
350234
}
351235
}
236+
added, deleted, updated = len(deletes), len(updates), len(inserts)
352237
return nil
353238
})
354239
if err != nil {
355240
return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
356241
}
357242

358-
log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
243+
log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated)
359244
return nil
360245
}
361246

‎services/context/repo.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,15 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
936936
ctx.ServerError("GetCommitsCount", err)
937937
return
938938
}
939+
if ctx.Repo.RefFullName.IsTag() {
940+
rel, err := repo_model.GetRelease(ctx, ctx.Repo.Repository.ID, ctx.Repo.RefFullName.TagName())
941+
if err == nil && rel.NumCommits <= 0 {
942+
rel.NumCommits = ctx.Repo.CommitsCount
943+
if err := repo_model.UpdateReleaseNumCommits(ctx, rel); err != nil {
944+
log.Error("UpdateReleaseNumCommits", err)
945+
}
946+
}
947+
}
939948
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
940949
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
941950
}

‎services/repository/migrate.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
149149
return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err)
150150
}
151151

152+
// if releases migration are not requested, we will sync all tags here
153+
// otherwise, the releases sync will be done out of this function
152154
if !opts.Releases {
153-
// note: this will greatly improve release (tag) sync
154-
// for pull-mirrors with many tags
155155
repo.IsMirror = opts.Mirror
156156
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
157157
log.Error("Failed to synchronize tags to releases for repository: %v", err)

‎services/repository/push.go‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ func pushDeleteBranch(ctx context.Context, repo *repo_model.Repository, pusher *
344344
// PushUpdateAddDeleteTags updates a number of added and delete tags
345345
func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error {
346346
return db.WithTx(ctx, func(ctx context.Context) error {
347-
if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil {
347+
if err := repo_model.PushUpdateDeleteTags(ctx, repo, delTags); err != nil {
348348
return err
349349
}
350350
return pushUpdateAddTags(ctx, repo, gitRepo, addTags)
@@ -415,11 +415,6 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
415415
createdAt = sig.When
416416
}
417417

418-
commitsCount, err := commit.CommitsCount()
419-
if err != nil {
420-
return fmt.Errorf("CommitsCount: %w", err)
421-
}
422-
423418
rel, has := relMap[lowerTag]
424419

425420
parts := strings.SplitN(tag.Message, "\n", 2)
@@ -435,7 +430,7 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
435430
LowerTagName: lowerTag,
436431
Target: "",
437432
Sha1: commit.ID.String(),
438-
NumCommits: commitsCount,
433+
NumCommits: -1, // the commits count will be updated when the UI needs it
439434
Note: note,
440435
IsDraft: false,
441436
IsPrerelease: false,
@@ -450,7 +445,6 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
450445
} else {
451446
rel.Sha1 = commit.ID.String()
452447
rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix())
453-
rel.NumCommits = commitsCount
454448
if rel.IsTag {
455449
rel.Title = parts[0]
456450
rel.Note = note

‎templates/repo/release/list.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
{{if $release.CreatedUnix}}
6262
<span class="time">{{DateUtils.TimeSince $release.CreatedUnix}}</span>
6363
{{end}}
64-
{{if and (not $release.IsDraft) ($.Permission.CanRead ctx.Consts.RepoUnitTypeCode)}}
64+
{{if and (gt $release.NumCommits 0) (not $release.IsDraft) ($.Permission.CanRead ctx.Consts.RepoUnitTypeCode)}}
6565
| <span class="ahead"><a href="{{$.RepoLink}}/compare/{{$release.TagName | PathEscapeSegments}}...{{$release.TargetBehind | PathEscapeSegments}}">{{ctx.Locale.Tr "repo.release.ahead.commits" $release.NumCommitsBehind}}</a> {{ctx.Locale.Tr "repo.release.ahead.target" $release.TargetBehind}}</span>
6666
{{end}}
6767
</p>

0 commit comments

Comments
 (0)