From b3a086bd76bd88ec72bb8b651ef8f1348b0698ab Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 12 Dec 2019 23:33:56 +0800 Subject: [PATCH 01/13] Sleep longer if request speed is over github limitation --- modules/migrations/base/downloader.go | 33 ++++++++++++++++++++++++--- modules/migrations/git.go | 10 ++++++++ modules/migrations/github.go | 31 ++++++++++++++++++++----- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index b853ec30209a1..980f2805bf381 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -21,6 +21,8 @@ type Downloader interface { GetIssues(page, perPage int) ([]*Issue, bool, error) GetComments(issueNumber int64) ([]*Comment, error) GetPullRequests(page, perPage int) ([]*PullRequest, error) + GetRequestTimes() int + GetRequestLimit() float32 } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader @@ -30,11 +32,16 @@ type DownloaderFactory interface { GitServiceType() structs.GitServiceType } +var ( + _ Downloader = &RetryDownloader{} +) + // RetryDownloader retry the downloads type RetryDownloader struct { Downloader - RetryTimes int // the total execute times - RetryDelay int // time to delay seconds + RetryTimes int // the total execute times + RetryDelay int // time to delay seconds + startTime time.Time // start time } // NewRetryDownloader creates a retry downloader @@ -43,7 +50,27 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr Downloader: downloader, RetryTimes: retryTimes, RetryDelay: retryDelay, + startTime: time.Now(), + } +} + +// GetRequestTimes returns request times +func (d *RetryDownloader) GetRequestTimes() int { + return d.Downloader.GetRequestTimes() +} + +// GetRequestLimit returns the limitation of the http request times per seconds. +func (d *RetryDownloader) GetRequestLimit() float32 { + return d.Downloader.GetRequestLimit() +} + +func (d *RetryDownloader) sleep() { + secs := int(time.Now().Sub(d.startTime) / time.Second) + var sleepTime = time.Second * time.Duration(d.RetryDelay) + if secs > 0 && float32(d.GetRequestTimes()/secs) > d.GetRequestLimit() { + sleepTime = time.Second * time.Duration(d.RetryDelay+int(float32(d.GetRequestTimes())/d.GetRequestLimit())-secs) } + time.Sleep(sleepTime) } // GetRepoInfo returns a repository information with retry @@ -57,7 +84,7 @@ func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { if repo, err = d.Downloader.GetRepoInfo(); err == nil { return repo, nil } - time.Sleep(time.Second * time.Duration(d.RetryDelay)) + d.sleep() } return nil, err } diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 75d05976cd687..5a84360a8babb 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -28,6 +28,16 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } +// GetRequestTimes returns request times to the git service +func (g *PlainGitDownloader) GetRequestTimes() int { + return 1 +} + +// GetRequestLimit returns the limitation of the http request times per seconds. +func (g *PlainGitDownloader) GetRequestLimit() float32 { + return 999999999.00 +} + // GetRepoInfo returns a repository information func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { // convert github repo to stand Repo diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 00d137a3de69a..b741003712853 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -67,12 +67,13 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { - ctx context.Context - client *github.Client - repoOwner string - repoName string - userName string - password string + ctx context.Context + client *github.Client + repoOwner string + repoName string + userName string + password string + requestTimes int } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -107,12 +108,23 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +// GetRequestTimes returns request times to the git service +func (g *GithubDownloaderV3) GetRequestTimes() int { + return g.requestTimes +} + +// GetRequestLimit returns the limitation of the http request times per seconds. +func (g *GithubDownloaderV3) GetRequestLimit() float32 { + return 5000 / 3600 +} + // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } + g.requestTimes++ // convert github repo to stand Repo return &base.Repository{ Owner: g.repoOwner, @@ -127,6 +139,7 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (g *GithubDownloaderV3) GetTopics() ([]string, error) { r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + g.requestTimes++ return r.Topics, err } @@ -142,6 +155,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { Page: i, PerPage: perPage, }}) + g.requestTimes++ if err != nil { return nil, err } @@ -194,6 +208,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { Page: i, PerPage: perPage, }) + g.requestTimes++ if err != nil { return nil, err } @@ -265,6 +280,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { Page: i, PerPage: perPage, }) + g.requestTimes++ if err != nil { return nil, err } @@ -306,6 +322,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, var allIssues = make([]*base.Issue, 0, perPage) issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + g.requestTimes++ if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -366,6 +383,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er } for { comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) + g.requestTimes++ if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -410,6 +428,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq var allPRs = make([]*base.PullRequest, 0, perPage) prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) + g.requestTimes++ if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } From 6c023ddbe8738b4dced25cf423d6f0ba36e9347c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 12 Dec 2019 23:53:01 +0800 Subject: [PATCH 02/13] improve code --- modules/migrations/base/downloader.go | 96 ++++++++++++++++++++++++--- modules/migrations/migrate.go | 2 + 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 980f2805bf381..0678165c6f8a5 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -33,7 +33,7 @@ type DownloaderFactory interface { } var ( - _ Downloader = &RetryDownloader{} + _ Downloader = &LimitDownloader{} ) // RetryDownloader retry the downloads @@ -64,15 +64,6 @@ func (d *RetryDownloader) GetRequestLimit() float32 { return d.Downloader.GetRequestLimit() } -func (d *RetryDownloader) sleep() { - secs := int(time.Now().Sub(d.startTime) / time.Second) - var sleepTime = time.Second * time.Duration(d.RetryDelay) - if secs > 0 && float32(d.GetRequestTimes()/secs) > d.GetRequestLimit() { - sleepTime = time.Second * time.Duration(d.RetryDelay+int(float32(d.GetRequestTimes())/d.GetRequestLimit())-secs) - } - time.Sleep(sleepTime) -} - // GetRepoInfo returns a repository information with retry func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { var ( @@ -84,7 +75,7 @@ func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { if repo, err = d.Downloader.GetRepoInfo(); err == nil { return repo, nil } - d.sleep() + time.Sleep(time.Second * time.Duration(d.RetryDelay)) } return nil, err } @@ -201,3 +192,86 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, er } return nil, err } + +var ( + _ Downloader = &LimitDownloader{} +) + +// LimitDownloader limit the downloads +type LimitDownloader struct { + Downloader + startTime time.Time // start time +} + +// NewLimitDownloader creates a limit downloader +func NewLimitDownloader(downloader Downloader) *LimitDownloader { + return &LimitDownloader{ + Downloader: downloader, + startTime: time.Now(), + } +} + +// GetRequestTimes returns request times +func (d *LimitDownloader) GetRequestTimes() int { + return d.Downloader.GetRequestTimes() +} + +// GetRequestLimit returns the limitation of the http request times per seconds. +func (d *LimitDownloader) GetRequestLimit() float32 { + return d.Downloader.GetRequestLimit() +} + +func (d *LimitDownloader) sleep() { + secs := int(time.Now().Sub(d.startTime) / time.Second) + if secs > 0 && float32(d.GetRequestTimes()/secs) > d.GetRequestLimit() { + time.Sleep(time.Second * time.Duration(int(float32(d.GetRequestTimes())/d.GetRequestLimit())-secs)) + } +} + +// GetRepoInfo returns a repository information with retry +func (d *LimitDownloader) GetRepoInfo() (*Repository, error) { + d.sleep() + return d.Downloader.GetRepoInfo() +} + +// GetTopics returns a repository's topics with retry +func (d *LimitDownloader) GetTopics() ([]string, error) { + d.sleep() + return d.Downloader.GetTopics() +} + +// GetMilestones returns a repository's milestones with retry +func (d *LimitDownloader) GetMilestones() ([]*Milestone, error) { + d.sleep() + return d.Downloader.GetMilestones() +} + +// GetReleases returns a repository's releases with retry +func (d *LimitDownloader) GetReleases() ([]*Release, error) { + d.sleep() + return d.Downloader.GetReleases() +} + +// GetLabels returns a repository's labels with retry +func (d *LimitDownloader) GetLabels() ([]*Label, error) { + d.sleep() + return d.Downloader.GetLabels() +} + +// GetIssues returns a repository's issues with retry +func (d *LimitDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { + d.sleep() + return d.Downloader.GetIssues(page, perPage) +} + +// GetComments returns a repository's comments with retry +func (d *LimitDownloader) GetComments(issueNumber int64) ([]*Comment, error) { + d.sleep() + return d.Downloader.GetComments(issueNumber) +} + +// GetPullRequests returns a repository's pull requests with retry +func (d *LimitDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) { + d.sleep() + return d.Downloader.GetPullRequests(page, perPage) +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 957d4c85d0c28..95e7a07993788 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -65,6 +65,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt uploader.gitServiceType = opts.GitServiceType + downloader = base.NewLimitDownloader(downloader) + if setting.Migrations.MaxAttempts > 1 { downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } From 75efaee58b5f6442ca1ac0cb34e722290e6fcdee Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 13 Dec 2019 00:09:12 +0800 Subject: [PATCH 03/13] remove unused code --- modules/migrations/base/downloader.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 0678165c6f8a5..0847101eb5913 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -39,9 +39,8 @@ var ( // RetryDownloader retry the downloads type RetryDownloader struct { Downloader - RetryTimes int // the total execute times - RetryDelay int // time to delay seconds - startTime time.Time // start time + RetryTimes int // the total execute times + RetryDelay int // time to delay seconds } // NewRetryDownloader creates a retry downloader @@ -50,7 +49,6 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr Downloader: downloader, RetryTimes: retryTimes, RetryDelay: retryDelay, - startTime: time.Now(), } } From 40f45ee7f2b13b563e09a1beebe9560ab59d8317 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 13 Dec 2019 00:11:34 +0800 Subject: [PATCH 04/13] fix lint --- modules/migrations/base/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 0847101eb5913..7009468e80893 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -220,7 +220,7 @@ func (d *LimitDownloader) GetRequestLimit() float32 { } func (d *LimitDownloader) sleep() { - secs := int(time.Now().Sub(d.startTime) / time.Second) + secs := int(time.Since(d.startTime) / time.Second) if secs > 0 && float32(d.GetRequestTimes()/secs) > d.GetRequestLimit() { time.Sleep(time.Second * time.Duration(int(float32(d.GetRequestTimes())/d.GetRequestLimit())-secs)) } From 3d9237e0545b526bd09d44305b0f57506fb21807 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 15:07:28 +0800 Subject: [PATCH 05/13] Use github's rate limit remain value to determine how long to sleep --- modules/migrations/base/downloader.go | 97 +-------------------------- modules/migrations/git.go | 10 --- modules/migrations/github.go | 72 +++++++++++--------- modules/migrations/migrate.go | 2 - 4 files changed, 40 insertions(+), 141 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 7009468e80893..06a947c83f91a 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -21,8 +21,6 @@ type Downloader interface { GetIssues(page, perPage int) ([]*Issue, bool, error) GetComments(issueNumber int64) ([]*Comment, error) GetPullRequests(page, perPage int) ([]*PullRequest, error) - GetRequestTimes() int - GetRequestLimit() float32 } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader @@ -33,7 +31,7 @@ type DownloaderFactory interface { } var ( - _ Downloader = &LimitDownloader{} + _ Downloader = &RetryDownloader{} ) // RetryDownloader retry the downloads @@ -52,16 +50,6 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr } } -// GetRequestTimes returns request times -func (d *RetryDownloader) GetRequestTimes() int { - return d.Downloader.GetRequestTimes() -} - -// GetRequestLimit returns the limitation of the http request times per seconds. -func (d *RetryDownloader) GetRequestLimit() float32 { - return d.Downloader.GetRequestLimit() -} - // GetRepoInfo returns a repository information with retry func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { var ( @@ -190,86 +178,3 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, er } return nil, err } - -var ( - _ Downloader = &LimitDownloader{} -) - -// LimitDownloader limit the downloads -type LimitDownloader struct { - Downloader - startTime time.Time // start time -} - -// NewLimitDownloader creates a limit downloader -func NewLimitDownloader(downloader Downloader) *LimitDownloader { - return &LimitDownloader{ - Downloader: downloader, - startTime: time.Now(), - } -} - -// GetRequestTimes returns request times -func (d *LimitDownloader) GetRequestTimes() int { - return d.Downloader.GetRequestTimes() -} - -// GetRequestLimit returns the limitation of the http request times per seconds. -func (d *LimitDownloader) GetRequestLimit() float32 { - return d.Downloader.GetRequestLimit() -} - -func (d *LimitDownloader) sleep() { - secs := int(time.Since(d.startTime) / time.Second) - if secs > 0 && float32(d.GetRequestTimes()/secs) > d.GetRequestLimit() { - time.Sleep(time.Second * time.Duration(int(float32(d.GetRequestTimes())/d.GetRequestLimit())-secs)) - } -} - -// GetRepoInfo returns a repository information with retry -func (d *LimitDownloader) GetRepoInfo() (*Repository, error) { - d.sleep() - return d.Downloader.GetRepoInfo() -} - -// GetTopics returns a repository's topics with retry -func (d *LimitDownloader) GetTopics() ([]string, error) { - d.sleep() - return d.Downloader.GetTopics() -} - -// GetMilestones returns a repository's milestones with retry -func (d *LimitDownloader) GetMilestones() ([]*Milestone, error) { - d.sleep() - return d.Downloader.GetMilestones() -} - -// GetReleases returns a repository's releases with retry -func (d *LimitDownloader) GetReleases() ([]*Release, error) { - d.sleep() - return d.Downloader.GetReleases() -} - -// GetLabels returns a repository's labels with retry -func (d *LimitDownloader) GetLabels() ([]*Label, error) { - d.sleep() - return d.Downloader.GetLabels() -} - -// GetIssues returns a repository's issues with retry -func (d *LimitDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { - d.sleep() - return d.Downloader.GetIssues(page, perPage) -} - -// GetComments returns a repository's comments with retry -func (d *LimitDownloader) GetComments(issueNumber int64) ([]*Comment, error) { - d.sleep() - return d.Downloader.GetComments(issueNumber) -} - -// GetPullRequests returns a repository's pull requests with retry -func (d *LimitDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) { - d.sleep() - return d.Downloader.GetPullRequests(page, perPage) -} diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 5a84360a8babb..75d05976cd687 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -28,16 +28,6 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } -// GetRequestTimes returns request times to the git service -func (g *PlainGitDownloader) GetRequestTimes() int { - return 1 -} - -// GetRequestLimit returns the limitation of the http request times per seconds. -func (g *PlainGitDownloader) GetRequestLimit() float32 { - return 999999999.00 -} - // GetRepoInfo returns a repository information func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { // convert github repo to stand Repo diff --git a/modules/migrations/github.go b/modules/migrations/github.go index b741003712853..649e319e1bb0c 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "strings" + "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -67,13 +68,13 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { - ctx context.Context - client *github.Client - repoOwner string - repoName string - userName string - password string - requestTimes int + ctx context.Context + client *github.Client + repoOwner string + repoName string + userName string + password string + remainRequests int } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -108,23 +109,20 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } -// GetRequestTimes returns request times to the git service -func (g *GithubDownloaderV3) GetRequestTimes() int { - return g.requestTimes -} - -// GetRequestLimit returns the limitation of the http request times per seconds. -func (g *GithubDownloaderV3) GetRequestLimit() float32 { - return 5000 / 3600 +func (g *GithubDownloaderV3) sleep() { + for g.remainRequests <= 0 { + time.Sleep(time.Second) + } } // GetRepoInfo returns a repository information func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { - gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + g.sleep() + gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } - g.requestTimes++ + g.remainRequests = resp.Rate.Remaining // convert github repo to stand Repo return &base.Repository{ Owner: g.repoOwner, @@ -138,9 +136,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { // GetTopics return github topics func (g *GithubDownloaderV3) GetTopics() ([]string, error) { - r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) - g.requestTimes++ - return r.Topics, err + g.sleep() + r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) + if err != nil { + return nil, err + } + g.remainRequests = resp.Rate.Remaining + return r.Topics, nil } // GetMilestones returns milestones @@ -148,17 +150,18 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { var perPage = 100 var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { - ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", ListOptions: github.ListOptions{ Page: i, PerPage: perPage, }}) - g.requestTimes++ if err != nil { return nil, err } + g.remainRequests = resp.Rate.Remaining for _, m := range ms { var desc string @@ -203,15 +206,16 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { var perPage = 100 var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { - ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, }) - g.requestTimes++ if err != nil { return nil, err } + g.remainRequests = resp.Rate.Remaining for _, label := range ls { labels = append(labels, convertGithubLabel(label)) @@ -275,15 +279,16 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { var perPage = 100 var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { - ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, + g.sleep() + ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, }) - g.requestTimes++ if err != nil { return nil, err } + g.remainRequests = resp.Rate.Remaining for _, release := range ls { releases = append(releases, g.convertGithubRelease(release)) @@ -320,12 +325,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } var allIssues = make([]*base.Issue, 0, perPage) - - issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) - g.requestTimes++ + g.sleep() + issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } + g.remainRequests = resp.Rate.Remaining for _, issue := range issues { if issue.IsPullRequest() { continue @@ -382,11 +387,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er }, } for { + g.sleep() comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) - g.requestTimes++ if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } + g.remainRequests = resp.Rate.Remaining for _, comment := range comments { var email string if comment.User.Email != nil { @@ -426,12 +432,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, } var allPRs = make([]*base.PullRequest, 0, perPage) - - prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) - g.requestTimes++ + g.sleep() + prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } + g.remainRequests = resp.Rate.Remaining for _, pr := range prs { var body string if pr.Body != nil { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 95e7a07993788..957d4c85d0c28 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -65,8 +65,6 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt uploader.gitServiceType = opts.GitServiceType - downloader = base.NewLimitDownloader(downloader) - if setting.Migrations.MaxAttempts > 1 { downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } From 181b595e5ca4108db0f488f2fbd9d202c9926bf6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 15:18:32 +0800 Subject: [PATCH 06/13] Save reset time when finished github api request --- modules/migrations/github.go | 41 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 649e319e1bb0c..f0c5ae60c1c67 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -68,13 +68,13 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { - ctx context.Context - client *github.Client - repoOwner string - repoName string - userName string - password string - remainRequests int + ctx context.Context + client *github.Client + repoOwner string + repoName string + userName string + password string + rate *github.Rate } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -110,8 +110,14 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith } func (g *GithubDownloaderV3) sleep() { - for g.remainRequests <= 0 { - time.Sleep(time.Second) + for g.rate.Remaining <= 0 { + time.Sleep(g.rate.Reset.Sub(time.Now())) + rates, _, err := g.client.RateLimits(g.ctx) + if err != nil { + log.Error("g.client.RateLimits: %s", err) + } + + g.rate = rates.GetCore() } } @@ -122,7 +128,8 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { if err != nil { return nil, err } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate + // convert github repo to stand Repo return &base.Repository{ Owner: g.repoOwner, @@ -141,7 +148,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) { if err != nil { return nil, err } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate return r.Topics, nil } @@ -161,7 +168,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { if err != nil { return nil, err } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, m := range ms { var desc string @@ -215,7 +222,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { if err != nil { return nil, err } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, label := range ls { labels = append(labels, convertGithubLabel(label)) @@ -288,7 +295,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { if err != nil { return nil, err } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, release := range ls { releases = append(releases, g.convertGithubRelease(release)) @@ -330,7 +337,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, issue := range issues { if issue.IsPullRequest() { continue @@ -392,7 +399,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, comment := range comments { var email string if comment.User.Email != nil { @@ -437,7 +444,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } - g.remainRequests = resp.Rate.Remaining + g.rate = &resp.Rate for _, pr := range prs { var body string if pr.Body != nil { From af548f223030eb5b34de2867ecbcf3c4f13bf080 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 15:27:37 +0800 Subject: [PATCH 07/13] fix bug --- modules/migrations/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index f0c5ae60c1c67..65f90ab701ac8 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -110,7 +110,7 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith } func (g *GithubDownloaderV3) sleep() { - for g.rate.Remaining <= 0 { + for g.rate != nil && g.rate.Remaining <= 0 { time.Sleep(g.rate.Reset.Sub(time.Now())) rates, _, err := g.client.RateLimits(g.ctx) if err != nil { From dc864841af71b92816c175f157436465dd06f1cf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 15:36:52 +0800 Subject: [PATCH 08/13] fix lint --- modules/migrations/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 65f90ab701ac8..7b8836e25c0e5 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -111,7 +111,7 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith func (g *GithubDownloaderV3) sleep() { for g.rate != nil && g.rate.Remaining <= 0 { - time.Sleep(g.rate.Reset.Sub(time.Now())) + time.Sleep(time.Until(g.rate.Reset.Time)) rates, _, err := g.client.RateLimits(g.ctx) if err != nil { log.Error("g.client.RateLimits: %s", err) From c735e2a63586dc166c62924a7ffd944e8c7e7764 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 16:27:53 +0800 Subject: [PATCH 09/13] Add context.Context for sleep --- modules/migrations/base/downloader.go | 7 +++++++ modules/migrations/git.go | 5 +++++ modules/migrations/gitea.go | 5 ++++- modules/migrations/github.go | 12 +++++++++++- modules/migrations/migrate.go | 7 +++++-- modules/task/migrate.go | 3 ++- routers/api/v1/repo/repo.go | 3 ++- 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 06a947c83f91a..87ade5c02e9e3 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -6,6 +6,7 @@ package base import ( + "context" "time" "code.gitea.io/gitea/modules/structs" @@ -13,6 +14,7 @@ import ( // Downloader downloads the site repo informations type Downloader interface { + SetContext(context.Context) GetRepoInfo() (*Repository, error) GetTopics() ([]string, error) GetMilestones() ([]*Milestone, error) @@ -50,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr } } +// SetContext set context +func (d *RetryDownloader) SetContext(ctx context.Context) { + d.Downloader.SetContext(ctx) +} + // GetRepoInfo returns a repository information with retry func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { var ( diff --git a/modules/migrations/git.go b/modules/migrations/git.go index 75d05976cd687..f65bf24a9843e 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -6,6 +6,7 @@ package migrations import ( "code.gitea.io/gitea/modules/migrations/base" + "context" ) var ( @@ -28,6 +29,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } +// SetContext set context +func (g *PlainGitDownloader) SetContext(ctx context.Context) { +} + // GetRepoInfo returns a repository information func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { // convert github repo to stand Repo diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index db2143fe7e9f2..f52f6c585ab27 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -6,6 +6,7 @@ package migrations import ( + "context" "fmt" "io" "net/http" @@ -35,6 +36,7 @@ var ( // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { + ctx context.Context doer *models.User repoOwner string repoName string @@ -49,8 +51,9 @@ type GiteaLocalUploader struct { } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 -func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { +func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { return &GiteaLocalUploader{ + ctx: ctx, doer: doer, repoOwner: repoOwner, repoName: repoName, diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 7b8836e25c0e5..7a21b290044ee 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -109,9 +109,19 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +// SetContext set context +func (g *GithubDownloaderV3) SetContext(ctx context.Context) { + g.ctx = ctx +} + func (g *GithubDownloaderV3) sleep() { for g.rate != nil && g.rate.Remaining <= 0 { - time.Sleep(time.Until(g.rate.Reset.Time)) + select { + case <-g.ctx.Done(): + return + case <-time.After(time.Now().Sub(g.rate.Reset.Time)): + } + rates, _, err := g.client.RateLimits(g.ctx) if err != nil { log.Error("g.client.RateLimits: %s", err) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 957d4c85d0c28..ece871a8574d3 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,6 +6,7 @@ package migrations import ( + "context" "fmt" "code.gitea.io/gitea/models" @@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { } // MigrateRepository migrate repository according MigrateOptions -func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { +func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) + uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) theFactory base.DownloaderFactory ) @@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } + downloader.SetContext(ctx) + if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) diff --git a/modules/task/migrate.go b/modules/task/migrate.go index 247403d7be355..d3b4fa45f01ab 100644 --- a/modules/task/migrate.go +++ b/modules/task/migrate.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" @@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) { } opts.MigrateToRepoID = t.RepoID - repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) + repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts) if err == nil { log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) return nil diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index cab0fc07e0e3a..be226c3438860 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" @@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } }() - if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil { + if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil { handleMigrateError(ctx, ctxUser, remoteAddr, err) return } From fad9df11f9115c3a3e236aecb8fddcf42142b540 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 18:30:55 +0800 Subject: [PATCH 10/13] fix test --- modules/migrations/gitea_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 73c119a15de74..438902f320ed1 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -10,6 +10,7 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) { var ( downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") - uploader = NewGiteaLocalUploader(user, user.Name, repoName) + uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ From b63bb4a5674fcc97e2a122f7168a4d53a6389646 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 23:41:46 +0800 Subject: [PATCH 11/13] improve code --- modules/migrations/github.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 7a21b290044ee..6858b9921f52a 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -116,10 +116,12 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) { func (g *GithubDownloaderV3) sleep() { for g.rate != nil && g.rate.Remaining <= 0 { + timer := time.NewTimer(time.Now().Sub(g.rate.Reset.Time)) select { case <-g.ctx.Done(): + timer.Stop() return - case <-time.After(time.Now().Sub(g.rate.Reset.Time)): + case <-timer.C: } rates, _, err := g.client.RateLimits(g.ctx) From 4d4023accc836ea3442ff9a69c808ef1224f613a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 16 Dec 2019 23:47:01 +0800 Subject: [PATCH 12/13] fix bug and lint --- modules/migrations/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 6858b9921f52a..fabdb4ae44547 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -116,7 +116,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) { func (g *GithubDownloaderV3) sleep() { for g.rate != nil && g.rate.Remaining <= 0 { - timer := time.NewTimer(time.Now().Sub(g.rate.Reset.Time)) + timer := time.NewTimer(time.Until(g.rate.Reset.Time)) select { case <-g.ctx.Done(): timer.Stop() From 07bbb07d51063d988a3e9dbecadb57db6b9d2427 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 17 Dec 2019 08:56:41 +0800 Subject: [PATCH 13/13] fix import order --- modules/migrations/git.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/migrations/git.go b/modules/migrations/git.go index f65bf24a9843e..f7b1e857e48a0 100644 --- a/modules/migrations/git.go +++ b/modules/migrations/git.go @@ -5,8 +5,9 @@ package migrations import ( - "code.gitea.io/gitea/modules/migrations/base" "context" + + "code.gitea.io/gitea/modules/migrations/base" ) var (