Skip to content

Commit b9fbab3

Browse files
harryzcydelvhzeripathlunny
authored and
dineshsalunke
committed
Implement sync push mirror on commit (go-gitea#19411)
Support synchronizing with the push mirrors whenever new commits are pushed or synced from pull mirror. Related Issues: go-gitea#18220 Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
1 parent c59d9c4 commit b9fbab3

File tree

12 files changed

+208
-98
lines changed

12 files changed

+208
-98
lines changed

‎models/migrations/migrations.go

+2
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,8 @@ var migrations = []Migration{
396396
NewMigration("Alter hook_task table TEXT fields to LONGTEXT", alterHookTaskTextFieldsToLongText),
397397
// v218 -> v219
398398
NewMigration("Improve Action table indices v2", improveActionTableIndices),
399+
// v219 -> v220
400+
NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror),
399401
}
400402

401403
// GetCurrentDBVersion returns the current db version

‎models/migrations/v219.go

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package migrations
6+
7+
import (
8+
"time"
9+
10+
"code.gitea.io/gitea/models/repo"
11+
"code.gitea.io/gitea/modules/timeutil"
12+
"xorm.io/xorm"
13+
)
14+
15+
func addSyncOnCommitColForPushMirror(x *xorm.Engine) error {
16+
type PushMirror struct {
17+
ID int64 `xorm:"pk autoincr"`
18+
RepoID int64 `xorm:"INDEX"`
19+
Repo *repo.Repository `xorm:"-"`
20+
RemoteName string
21+
22+
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
23+
Interval time.Duration
24+
CreatedUnix timeutil.TimeStamp `xorm:"created"`
25+
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
26+
LastError string `xorm:"text"`
27+
}
28+
29+
return x.Sync2(new(PushMirror))
30+
}

‎models/repo/pushmirror.go

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type PushMirror struct {
2323
Repo *Repository `xorm:"-"`
2424
RemoteName string
2525

26+
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
2627
Interval time.Duration
2728
CreatedUnix timeutil.TimeStamp `xorm:"created"`
2829
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
@@ -93,6 +94,14 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
9394
return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors)
9495
}
9596

97+
// GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits
98+
func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) {
99+
mirrors := make([]*PushMirror, 0, 10)
100+
return mirrors, db.GetEngine(db.DefaultContext).
101+
Where("repo_id=? AND sync_on_commit=?", repoID, true).
102+
Find(&mirrors)
103+
}
104+
96105
// PushMirrorsIterate iterates all push-mirror repositories.
97106
func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
98107
return db.GetEngine(db.DefaultContext).

‎modules/mirror/mirror.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package mirror
6+
7+
import (
8+
"code.gitea.io/gitea/modules/graceful"
9+
"code.gitea.io/gitea/modules/log"
10+
"code.gitea.io/gitea/modules/queue"
11+
"code.gitea.io/gitea/modules/setting"
12+
)
13+
14+
var mirrorQueue queue.UniqueQueue
15+
16+
// SyncType type of sync request
17+
type SyncType int
18+
19+
const (
20+
// PullMirrorType for pull mirrors
21+
PullMirrorType SyncType = iota
22+
// PushMirrorType for push mirrors
23+
PushMirrorType
24+
)
25+
26+
// SyncRequest for the mirror queue
27+
type SyncRequest struct {
28+
Type SyncType
29+
ReferenceID int64 // RepoID for pull mirror, MirrorID for push mirror
30+
}
31+
32+
// StartSyncMirrors starts a go routine to sync the mirrors
33+
func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) {
34+
if !setting.Mirror.Enabled {
35+
return
36+
}
37+
mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest))
38+
39+
go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run)
40+
}
41+
42+
// AddPullMirrorToQueue adds repoID to mirror queue
43+
func AddPullMirrorToQueue(repoID int64) {
44+
addMirrorToQueue(PullMirrorType, repoID)
45+
}
46+
47+
// AddPushMirrorToQueue adds the push mirror to the queue
48+
func AddPushMirrorToQueue(mirrorID int64) {
49+
addMirrorToQueue(PushMirrorType, mirrorID)
50+
}
51+
52+
func addMirrorToQueue(syncType SyncType, referenceID int64) {
53+
if !setting.Mirror.Enabled {
54+
return
55+
}
56+
go func() {
57+
if err := PushToQueue(syncType, referenceID); err != nil {
58+
log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]. Error: %v", referenceID, err)
59+
}
60+
}()
61+
}
62+
63+
// PushToQueue adds the sync request to the queue
64+
func PushToQueue(mirrorType SyncType, referenceID int64) error {
65+
return mirrorQueue.Push(&SyncRequest{
66+
Type: mirrorType,
67+
ReferenceID: referenceID,
68+
})
69+
}

‎modules/notification/mirror/mirror.go

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package mirror
6+
7+
import (
8+
repo_model "code.gitea.io/gitea/models/repo"
9+
user_model "code.gitea.io/gitea/models/user"
10+
"code.gitea.io/gitea/modules/log"
11+
mirror_module "code.gitea.io/gitea/modules/mirror"
12+
"code.gitea.io/gitea/modules/notification/base"
13+
"code.gitea.io/gitea/modules/repository"
14+
)
15+
16+
type mirrorNotifier struct {
17+
base.NullNotifier
18+
}
19+
20+
var _ base.Notifier = &mirrorNotifier{}
21+
22+
// NewNotifier create a new mirrorNotifier notifier
23+
func NewNotifier() base.Notifier {
24+
return &mirrorNotifier{}
25+
}
26+
27+
func (m *mirrorNotifier) NotifyPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
28+
syncPushMirrorWithSyncOnCommit(repo.ID)
29+
}
30+
31+
func (m *mirrorNotifier) NotifySyncPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) {
32+
syncPushMirrorWithSyncOnCommit(repo.ID)
33+
}
34+
35+
func syncPushMirrorWithSyncOnCommit(repoID int64) {
36+
pushMirrors, err := repo_model.GetPushMirrorsSyncedOnCommit(repoID)
37+
if err != nil {
38+
log.Error("repo_model.GetPushMirrorsSyncedOnCommit failed: %v", err)
39+
return
40+
}
41+
42+
for _, mirror := range pushMirrors {
43+
mirror_module.AddPushMirrorToQueue(mirror.ID)
44+
}
45+
}

‎modules/notification/notification.go

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"code.gitea.io/gitea/modules/notification/base"
1515
"code.gitea.io/gitea/modules/notification/indexer"
1616
"code.gitea.io/gitea/modules/notification/mail"
17+
"code.gitea.io/gitea/modules/notification/mirror"
1718
"code.gitea.io/gitea/modules/notification/ui"
1819
"code.gitea.io/gitea/modules/notification/webhook"
1920
"code.gitea.io/gitea/modules/repository"
@@ -37,6 +38,7 @@ func NewContext() {
3738
RegisterNotifier(indexer.NewNotifier())
3839
RegisterNotifier(webhook.NewNotifier())
3940
RegisterNotifier(action.NewNotifier())
41+
RegisterNotifier(mirror.NewNotifier())
4042
}
4143

4244
// NotifyCreateIssueComment notifies issue comment related message to notifiers

‎options/locale/locale_en-US.ini

+2-1
Original file line numberDiff line numberDiff line change
@@ -861,8 +861,9 @@ default_branch = Default Branch
861861
default_branch_helper = The default branch is the base branch for pull requests and code commits.
862862
mirror_prune = Prune
863863
mirror_prune_desc = Remove obsolete remote-tracking references
864-
mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable automatic sync. (Minimum interval: %s)
864+
mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable periodic sync. (Minimum interval: %s)
865865
mirror_interval_invalid = The mirror interval is not valid.
866+
mirror_sync_on_commit = Sync when commits are pushed
866867
mirror_address = Clone From URL
867868
mirror_address_desc = Put any required credentials in the Authorization section.
868869
mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly.

‎routers/api/v1/repo/mirror.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ import (
1111
repo_model "code.gitea.io/gitea/models/repo"
1212
"code.gitea.io/gitea/models/unit"
1313
"code.gitea.io/gitea/modules/context"
14+
mirror_module "code.gitea.io/gitea/modules/mirror"
1415
"code.gitea.io/gitea/modules/setting"
15-
mirror_service "code.gitea.io/gitea/services/mirror"
1616
)
1717

1818
// MirrorSync adds a mirrored repository to the sync queue
@@ -59,7 +59,7 @@ func MirrorSync(ctx *context.APIContext) {
5959
return
6060
}
6161

62-
mirror_service.StartToMirror(repo.ID)
62+
mirror_module.AddPullMirrorToQueue(repo.ID)
6363

6464
ctx.Status(http.StatusOK)
6565
}

‎routers/web/repo/setting.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"code.gitea.io/gitea/modules/indexer/stats"
3030
"code.gitea.io/gitea/modules/lfs"
3131
"code.gitea.io/gitea/modules/log"
32+
mirror_module "code.gitea.io/gitea/modules/mirror"
3233
"code.gitea.io/gitea/modules/repository"
3334
"code.gitea.io/gitea/modules/setting"
3435
"code.gitea.io/gitea/modules/structs"
@@ -272,7 +273,7 @@ func SettingsPost(ctx *context.Context) {
272273
return
273274
}
274275

275-
mirror_service.StartToMirror(repo.ID)
276+
mirror_module.AddPullMirrorToQueue(repo.ID)
276277

277278
ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
278279
ctx.Redirect(repo.Link() + "/settings")
@@ -289,7 +290,7 @@ func SettingsPost(ctx *context.Context) {
289290
return
290291
}
291292

292-
mirror_service.AddPushMirrorToQueue(m.ID)
293+
mirror_module.AddPushMirrorToQueue(m.ID)
293294

294295
ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress"))
295296
ctx.Redirect(repo.Link() + "/settings")
@@ -357,10 +358,11 @@ func SettingsPost(ctx *context.Context) {
357358
}
358359

359360
m := &repo_model.PushMirror{
360-
RepoID: repo.ID,
361-
Repo: repo,
362-
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
363-
Interval: interval,
361+
RepoID: repo.ID,
362+
Repo: repo,
363+
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
364+
SyncOnCommit: form.PushMirrorSyncOnCommit,
365+
Interval: interval,
364366
}
365367
if err := repo_model.InsertPushMirror(m); err != nil {
366368
ctx.ServerError("InsertPushMirror", err)

‎services/forms/repo_form.go

+18-17
Original file line numberDiff line numberDiff line change
@@ -115,23 +115,24 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, err
115115

116116
// RepoSettingForm form for changing repository settings
117117
type RepoSettingForm struct {
118-
RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"`
119-
Description string `binding:"MaxSize(255)"`
120-
Website string `binding:"ValidUrl;MaxSize(255)"`
121-
Interval string
122-
MirrorAddress string
123-
MirrorUsername string
124-
MirrorPassword string
125-
LFS bool `form:"mirror_lfs"`
126-
LFSEndpoint string `form:"mirror_lfs_endpoint"`
127-
PushMirrorID string
128-
PushMirrorAddress string
129-
PushMirrorUsername string
130-
PushMirrorPassword string
131-
PushMirrorInterval string
132-
Private bool
133-
Template bool
134-
EnablePrune bool
118+
RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"`
119+
Description string `binding:"MaxSize(255)"`
120+
Website string `binding:"ValidUrl;MaxSize(255)"`
121+
Interval string
122+
MirrorAddress string
123+
MirrorUsername string
124+
MirrorPassword string
125+
LFS bool `form:"mirror_lfs"`
126+
LFSEndpoint string `form:"mirror_lfs_endpoint"`
127+
PushMirrorID string
128+
PushMirrorAddress string
129+
PushMirrorUsername string
130+
PushMirrorPassword string
131+
PushMirrorSyncOnCommit bool
132+
PushMirrorInterval string
133+
Private bool
134+
Template bool
135+
EnablePrune bool
135136

136137
// Advanced settings
137138
EnableWiki bool

0 commit comments

Comments
 (0)