From 938026f414b5566ec20fb21065fd2b1a6cf63ae7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 3 Feb 2019 23:00:14 +0000 Subject: [PATCH 01/10] Move migration clone to goroutine The current code places the git clone in the POST goroutine, blocking that goroutine until it is finished. This PR asynchronises this, placing the clone within its own goroutine. Fix #3770 --- models/action.go | 43 ++--- models/release_test.go | 4 +- models/repo.go | 242 +++++++++++++++++++--------- modules/templates/helper.go | 14 +- options/locale/locale_en-US.ini | 3 + routers/api/v1/repo/repo.go | 4 +- routers/repo/repo.go | 14 +- templates/user/dashboard/feeds.tmpl | 6 + 8 files changed, 221 insertions(+), 109 deletions(-) diff --git a/models/action.go b/models/action.go index 5dc0eb18e699e..0972cef2d9a68 100644 --- a/models/action.go +++ b/models/action.go @@ -30,26 +30,29 @@ type ActionType int // Possible action types. const ( - ActionCreateRepo ActionType = iota + 1 // 1 - ActionRenameRepo // 2 - ActionStarRepo // 3 - ActionWatchRepo // 4 - ActionCommitRepo // 5 - ActionCreateIssue // 6 - ActionCreatePullRequest // 7 - ActionTransferRepo // 8 - ActionPushTag // 9 - ActionCommentIssue // 10 - ActionMergePullRequest // 11 - ActionCloseIssue // 12 - ActionReopenIssue // 13 - ActionClosePullRequest // 14 - ActionReopenPullRequest // 15 - ActionDeleteTag // 16 - ActionDeleteBranch // 17 - ActionMirrorSyncPush // 18 - ActionMirrorSyncCreate // 19 - ActionMirrorSyncDelete // 20 + ActionCreateRepo ActionType = iota + 1 // 1 + ActionRenameRepo // 2 + ActionStarRepo // 3 + ActionWatchRepo // 4 + ActionCommitRepo // 5 + ActionCreateIssue // 6 + ActionCreatePullRequest // 7 + ActionTransferRepo // 8 + ActionPushTag // 9 + ActionCommentIssue // 10 + ActionMergePullRequest // 11 + ActionCloseIssue // 12 + ActionReopenIssue // 13 + ActionClosePullRequest // 14 + ActionReopenPullRequest // 15 + ActionDeleteTag // 16 + ActionDeleteBranch // 17 + ActionMirrorSyncPush // 18 + ActionMirrorSyncCreate // 19 + ActionMirrorSyncDelete // 20 + ActionMigrationStarted // 21 + ActionMigrationSuccessful // 22 + ActionMigrationFailure // 23 ) var ( diff --git a/models/release_test.go b/models/release_test.go index de34109e8c88b..1b665ffd57d85 100644 --- a/models/release_test.go +++ b/models/release_test.go @@ -108,7 +108,9 @@ func TestRelease_MirrorDelete(t *testing.T) { IsMirror: true, RemoteAddr: repoPath, } - mirror, err := MigrateRepository(user, user, migrationOptions) + mirror, err := MigrateRepository(user, user, migrationOptions, func(err error) string { + return err.Error() + }) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) diff --git a/models/repo.go b/models/repo.go index 873fd407fb931..6946d04aa8db0 100644 --- a/models/repo.go +++ b/models/repo.go @@ -896,115 +896,198 @@ func wikiRemoteURL(remote string) string { } // MigrateRepository migrates a existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { +func MigrateRepository(doer, u *User, opts MigrateRepoOptions, messageConverter func(error) string) (*Repository, error) { repo, err := CreateRepository(doer, u, CreateRepoOptions{ Name: opts.Name, Description: opts.Description, IsPrivate: opts.IsPrivate, IsMirror: opts.IsMirror, + NoWatchers: true, }) if err != nil { return nil, err } - repoPath := RepoPath(u.Name, opts.Name) - wikiPath := WikiPath(u.Name, opts.Name) - - if u.IsOrganization() { - t, err := u.GetOwnerTeam() - if err != nil { - return nil, err + env, ok := os.LookupEnv("GIT_TERMINAL_PROMPT=0") + os.Setenv("GIT_TERMINAL_PROMPT", "0") + if _, err = git.NewCommand("ls-remote", "-h", opts.RemoteAddr).RunTimeout(1 * time.Minute); err != nil { + if ok { + os.Setenv("GIT_TERMINAL_PROMPT", env) + } else { + os.Unsetenv("GIT_TERMINAL_PROMPT") } - repo.NumWatches = t.NumMembers - } else { - repo.NumWatches = 1 + return repo, fmt.Errorf("Clone: %v", err) } + if ok { + os.Setenv("GIT_TERMINAL_PROMPT", env) + } else { + os.Unsetenv("GIT_TERMINAL_PROMPT") + } + + // OK if we succeeded above then we know that the clone should start... + go func() { + repoPath := RepoPath(u.Name, opts.Name) + wikiPath := WikiPath(u.Name, opts.Name) + + failedMigration := func(err error) { + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationFailure, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + Content: messageConverter(err), + }) - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + if repo != nil { + if errDelete := DeleteRepository(doer, u.ID, repo.ID); errDelete != nil { + log.Error(4, "DeleteRepository: %v", errDelete) + } + } - if err := os.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) - } + } + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationStarted, + RepoID: repo.ID, + Repo: repo, + Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), + IsPrivate: repo.IsPrivate, + }) + repo.IsArchived = true + if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { + failedMigration(err) + return + } - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - }); err != nil { - return repo, fmt.Errorf("Clone: %v", err) - } + if u.IsOrganization() { + t, err := u.GetOwnerTeam() + if err != nil { + failedMigration(err) + return + } + repo.NumWatches = t.NumMembers + } else { + repo.NumWatches = 1 + } - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) - if len(wikiRemotePath) > 0 { - if err := os.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + + if err := os.RemoveAll(repoPath); err != nil { + failedMigration(fmt.Errorf("Failed to remove %s: %v", repoPath, err)) + return } - if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, - Branch: "master", }); err != nil { - log.Warn("Clone wiki: %v", err) + failedMigration(fmt.Errorf("Clone: %v", err)) + return + + } + + wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) + return } - } - } - // Check if repository is empty. - _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") - if err != nil { - if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { - repo.IsEmpty = true - } else { - return repo, fmt.Errorf("check empty: %v - %s", err, stderr) + if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) + return + } + } } - } - if !repo.IsEmpty { - // Try to get HEAD branch and set it as default branch. - gitRepo, err := git.OpenRepository(repoPath) + // Check if repository is empty. + _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") if err != nil { - return repo, fmt.Errorf("OpenRepository: %v", err) - } - headBranch, err := gitRepo.GetHEADBranch() - if err != nil { - return repo, fmt.Errorf("GetHEADBranch: %v", err) + if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { + repo.IsEmpty = true + } else { + failedMigration(fmt.Errorf("check empty: %v - %s", err, stderr)) + return + } } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name + + if !repo.IsEmpty { + // Try to get HEAD branch and set it as default branch. + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + failedMigration(fmt.Errorf("OpenRepository: %v", err)) + return + } + headBranch, err := gitRepo.GetHEADBranch() + if err != nil { + failedMigration(fmt.Errorf("GetHEADBranch: %v", err)) + return + } + if headBranch != nil { + repo.DefaultBranch = headBranch.Name + } + + if err = SyncReleasesWithTags(repo, gitRepo); err != nil { + log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) + } } - if err = SyncReleasesWithTags(repo, gitRepo); err != nil { - log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) + if err = repo.UpdateSize(); err != nil { + log.Error(4, "Failed to update size for repository: %v", err) } - } - if err = repo.UpdateSize(); err != nil { - log.Error(4, "Failed to update size for repository: %v", err) - } + if opts.IsMirror { + if _, err = x.InsertOne(&Mirror{ + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + EnablePrune: true, + NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + }); err != nil { + failedMigration(fmt.Errorf("InsertOne: %v", err)) + return + } - if opts.IsMirror { - if _, err = x.InsertOne(&Mirror{ - RepoID: repo.ID, - Interval: setting.Mirror.DefaultInterval, - EnablePrune: true, - NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), - }); err != nil { - return repo, fmt.Errorf("InsertOne: %v", err) + repo.IsMirror = true + err = UpdateRepository(repo, false) + } else { + repo, err = CleanUpMigrateInfo(repo) } - repo.IsMirror = true - err = UpdateRepository(repo, false) - } else { - repo, err = CleanUpMigrateInfo(repo) - } + if err != nil { + if !repo.IsEmpty { + UpdateRepoIndexer(repo) + } + failedMigration(err) + return + } - if err != nil && !repo.IsEmpty { - UpdateRepoIndexer(repo) - } + repo.IsArchived = false + if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { + failedMigration(err) + return + } + + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationSuccessful, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), + }) + }() return repo, err } @@ -1120,6 +1203,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + NoWatchers bool } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -1273,7 +1357,7 @@ func IsUsableRepoName(name string) error { return isUsableName(reservedRepoNames, reservedRepoPatterns, name) } -func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err error) { +func createRepository(e *xorm.Session, doer, u *User, repo *Repository, noWatchers bool) (err error) { if err = IsUsableRepoName(repo.Name); err != nil { return err } @@ -1359,10 +1443,12 @@ func createRepository(e *xorm.Session, doer, u *User, repo *Repository) (err err return fmt.Errorf("watchRepo: %v", err) } } - if err = newRepoAction(e, doer, repo); err != nil { - return fmt.Errorf("newRepoAction: %v", err) - } + if !noWatchers { + if err = newRepoAction(e, doer, repo); err != nil { + return fmt.Errorf("newRepoAction: %v", err) + } + } return nil } @@ -1388,7 +1474,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err return nil, err } - if err = createRepository(sess, doer, u, repo); err != nil { + if err = createRepository(sess, doer, u, repo, opts.NoWatchers); err != nil { return nil, err } @@ -2420,7 +2506,7 @@ func ForkRepository(doer, u *User, oldRepo *Repository, name, desc string) (_ *R return nil, err } - if err = createRepository(sess, doer, u, repo); err != nil { + if err = createRepository(sess, doer, u, repo, true); err != nil { return nil, err } diff --git a/modules/templates/helper.go b/modules/templates/helper.go index ce077d1a928e7..2158e48930ab6 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -104,12 +104,12 @@ func NewFuncMap() []template.FuncMap { } return str[start:end] }, - "EllipsisString": base.EllipsisString, - "DiffTypeToStr": DiffTypeToStr, - "DiffLineTypeToStr": DiffLineTypeToStr, - "Sha1": Sha1, - "ShortSha": base.ShortSha, - "MD5": base.EncodeMD5, + "EllipsisString": base.EllipsisString, + "DiffTypeToStr": DiffTypeToStr, + "DiffLineTypeToStr": DiffLineTypeToStr, + "Sha1": Sha1, + "ShortSha": base.ShortSha, + "MD5": base.EncodeMD5, "ActionContent2Commits": ActionContent2Commits, "PathEscape": url.PathEscape, "EscapePound": func(str string) string { @@ -414,7 +414,7 @@ func ActionIcon(opType models.ActionType) string { return "issue-closed" case models.ActionReopenIssue, models.ActionReopenPullRequest: return "issue-reopened" - case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete: + case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete, models.ActionMigrationStarted, models.ActionMigrationSuccessful, models.ActionMigrationFailure: return "repo-clone" default: return "invalid type" diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2d32fac9c7342..5cab6b99d034d 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1714,6 +1714,9 @@ compare_commits = Compare %d commits mirror_sync_push = synced commits to %[3]s at %[4]s from mirror mirror_sync_create = synced new reference %[2]s to %[3]s from mirror mirror_sync_delete = synced and deleted reference %[2]s at %[3]s from mirror +migration_started = started a migration from %s to %s +migration_successful = successfully migrated from %s to %s +migration_failure = unsuccessfully migrated to %s: %s [tool] ago = %s ago diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index d78700c6b0b58..084a9a3761b26 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -398,6 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { IsPrivate: form.Private || setting.Repository.ForcePrivate, IsMirror: form.Mirror, RemoteAddr: remoteAddr, + }, func(err error) string { + return util.URLSanitizedError(err, remoteAddr).Error() }) if err != nil { err = util.URLSanitizedError(err, remoteAddr) @@ -410,7 +412,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { return } - log.Trace("Repository migrated: %s/%s", ctxUser.Name, form.RepoName) + log.Trace("Repository migration started: %s/%s", ctxUser.Name, form.RepoName) ctx.JSON(201, repo.APIFormat(models.AccessModeAdmin)) } diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 960961a5e569c..1df4ff8e58cbd 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -249,10 +249,20 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { IsPrivate: form.Private || setting.Repository.ForcePrivate, IsMirror: form.Mirror, RemoteAddr: remoteAddr, + }, func(err error) string { + err = util.URLSanitizedError(err, remoteAddr) + + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return ctx.Tr("form.auth_failed", err.Error()) + } else if strings.Contains(err.Error(), "fatal:") { + return ctx.Tr("repo.migrate.failed", err.Error()) + } + return err.Error() }) if err == nil { - log.Trace("Repository migrated [%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName) - ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName) + log.Trace("Repository migration started [%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName) + ctx.Redirect(setting.AppURL) return } diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index 849ad7fb2f28f..804168cea26ab 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -57,6 +57,12 @@ {{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} {{else if eq .GetOpType 20}} {{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 21}} + {{$.i18n.Tr "action.migration_started" .GetContent .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 22}} + {{$.i18n.Tr "action.migration_successful" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 23}} + {{$.i18n.Tr "action.migration_failure" .ShortRepoPath .GetContent | Str2html}} {{end}}

{{if or (eq .GetOpType 5) (eq .GetOpType 18)}} From e6b71bfca3eecea8cbca099e4e9e1f34a21741ff Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Feb 2019 18:10:38 +0000 Subject: [PATCH 02/10] Add panic recovery --- models/repo.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/models/repo.go b/models/repo.go index 6946d04aa8db0..ac110af42baeb 100644 --- a/models/repo.go +++ b/models/repo.go @@ -947,6 +947,16 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions, messageConverter } } + + defer func() { + if err := recover(); err != nil { + log.Error(3, "PANIC: Migration failed with panic: %v", err) + + // fail the migration + failedMigration(fmt.Errorf("Migration failed: %v", err)) + } + }() + NotifyWatchers(&Action{ ActUserID: doer.ID, ActUser: doer, From b43d843d24c9c05d72cf36ed417c58547d0b047b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Feb 2019 18:20:53 +0000 Subject: [PATCH 03/10] Fix fmt --- modules/templates/helper.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 2158e48930ab6..fbb0af370b9fe 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -104,12 +104,12 @@ func NewFuncMap() []template.FuncMap { } return str[start:end] }, - "EllipsisString": base.EllipsisString, - "DiffTypeToStr": DiffTypeToStr, - "DiffLineTypeToStr": DiffLineTypeToStr, - "Sha1": Sha1, - "ShortSha": base.ShortSha, - "MD5": base.EncodeMD5, + "EllipsisString": base.EllipsisString, + "DiffTypeToStr": DiffTypeToStr, + "DiffLineTypeToStr": DiffLineTypeToStr, + "Sha1": Sha1, + "ShortSha": base.ShortSha, + "MD5": base.EncodeMD5, "ActionContent2Commits": ActionContent2Commits, "PathEscape": url.PathEscape, "EscapePound": func(str string) string { From b120e6cc797c30eb4f1c856d9d2136e0304d999f Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Feb 2019 19:43:28 +0000 Subject: [PATCH 04/10] Move migration functions out of models/repo.go --- models/repo.go | 274 -------------------------- models/repo_migrate.go | 289 ++++++++++++++++++++++++++++ options/locale/locale_en-US.ini | 2 +- templates/user/dashboard/feeds.tmpl | 2 +- 4 files changed, 291 insertions(+), 276 deletions(-) create mode 100644 models/repo_migrate.go diff --git a/models/repo.go b/models/repo.go index ac110af42baeb..fc8b4c3c216c2 100644 --- a/models/repo.go +++ b/models/repo.go @@ -36,7 +36,6 @@ import ( "github.com/go-xorm/builder" "github.com/go-xorm/xorm" version "github.com/mcuadros/go-version" - ini "gopkg.in/ini.v1" ) var repoWorkingPool = sync.NewExclusivePool() @@ -867,255 +866,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -// MigrateRepoOptions contains the repository migrate options -type MigrateRepoOptions struct { - Name string - Description string - IsPrivate bool - IsMirror bool - RemoteAddr string -} - -/* - GitHub, GitLab, Gogs: *.wiki.git - BitBucket: *.git/wiki -*/ -var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} - -// wikiRemoteURL returns accessible repository URL for wiki if exists. -// Otherwise, it returns an empty string. -func wikiRemoteURL(remote string) string { - remote = strings.TrimSuffix(remote, ".git") - for _, suffix := range commonWikiURLSuffixes { - wikiURL := remote + suffix - if git.IsRepoURLAccessible(wikiURL) { - return wikiURL - } - } - return "" -} - -// MigrateRepository migrates a existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions, messageConverter func(error) string) (*Repository, error) { - repo, err := CreateRepository(doer, u, CreateRepoOptions{ - Name: opts.Name, - Description: opts.Description, - IsPrivate: opts.IsPrivate, - IsMirror: opts.IsMirror, - NoWatchers: true, - }) - if err != nil { - return nil, err - } - - env, ok := os.LookupEnv("GIT_TERMINAL_PROMPT=0") - os.Setenv("GIT_TERMINAL_PROMPT", "0") - if _, err = git.NewCommand("ls-remote", "-h", opts.RemoteAddr).RunTimeout(1 * time.Minute); err != nil { - if ok { - os.Setenv("GIT_TERMINAL_PROMPT", env) - } else { - os.Unsetenv("GIT_TERMINAL_PROMPT") - } - return repo, fmt.Errorf("Clone: %v", err) - } - if ok { - os.Setenv("GIT_TERMINAL_PROMPT", env) - } else { - os.Unsetenv("GIT_TERMINAL_PROMPT") - } - - // OK if we succeeded above then we know that the clone should start... - go func() { - repoPath := RepoPath(u.Name, opts.Name) - wikiPath := WikiPath(u.Name, opts.Name) - - failedMigration := func(err error) { - NotifyWatchers(&Action{ - ActUserID: doer.ID, - ActUser: doer, - OpType: ActionMigrationFailure, - RepoID: repo.ID, - Repo: repo, - IsPrivate: repo.IsPrivate, - Content: messageConverter(err), - }) - - if repo != nil { - if errDelete := DeleteRepository(doer, u.ID, repo.ID); errDelete != nil { - log.Error(4, "DeleteRepository: %v", errDelete) - } - } - - } - - defer func() { - if err := recover(); err != nil { - log.Error(3, "PANIC: Migration failed with panic: %v", err) - - // fail the migration - failedMigration(fmt.Errorf("Migration failed: %v", err)) - } - }() - - NotifyWatchers(&Action{ - ActUserID: doer.ID, - ActUser: doer, - OpType: ActionMigrationStarted, - RepoID: repo.ID, - Repo: repo, - Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), - IsPrivate: repo.IsPrivate, - }) - repo.IsArchived = true - if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { - failedMigration(err) - return - } - - if u.IsOrganization() { - t, err := u.GetOwnerTeam() - if err != nil { - failedMigration(err) - return - } - repo.NumWatches = t.NumMembers - } else { - repo.NumWatches = 1 - } - - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - - if err := os.RemoveAll(repoPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", repoPath, err)) - return - } - - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - }); err != nil { - failedMigration(fmt.Errorf("Clone: %v", err)) - return - - } - - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) - if len(wikiRemotePath) > 0 { - if err := os.RemoveAll(wikiPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) - return - } - - if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - }); err != nil { - log.Warn("Clone wiki: %v", err) - if err := os.RemoveAll(wikiPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) - return - } - } - } - - // Check if repository is empty. - _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") - if err != nil { - if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { - repo.IsEmpty = true - } else { - failedMigration(fmt.Errorf("check empty: %v - %s", err, stderr)) - return - } - } - - if !repo.IsEmpty { - // Try to get HEAD branch and set it as default branch. - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - failedMigration(fmt.Errorf("OpenRepository: %v", err)) - return - } - headBranch, err := gitRepo.GetHEADBranch() - if err != nil { - failedMigration(fmt.Errorf("GetHEADBranch: %v", err)) - return - } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name - } - - if err = SyncReleasesWithTags(repo, gitRepo); err != nil { - log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) - } - } - - if err = repo.UpdateSize(); err != nil { - log.Error(4, "Failed to update size for repository: %v", err) - } - - if opts.IsMirror { - if _, err = x.InsertOne(&Mirror{ - RepoID: repo.ID, - Interval: setting.Mirror.DefaultInterval, - EnablePrune: true, - NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), - }); err != nil { - failedMigration(fmt.Errorf("InsertOne: %v", err)) - return - } - - repo.IsMirror = true - err = UpdateRepository(repo, false) - } else { - repo, err = CleanUpMigrateInfo(repo) - } - - if err != nil { - if !repo.IsEmpty { - UpdateRepoIndexer(repo) - } - failedMigration(err) - return - } - - repo.IsArchived = false - if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { - failedMigration(err) - return - } - - NotifyWatchers(&Action{ - ActUserID: doer.ID, - ActUser: doer, - OpType: ActionMigrationSuccessful, - RepoID: repo.ID, - Repo: repo, - IsPrivate: repo.IsPrivate, - Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), - }) - }() - - return repo, err -} - -// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". -// This also removes possible user credentials. -func cleanUpMigrateGitConfig(configPath string) error { - cfg, err := ini.Load(configPath) - if err != nil { - return fmt.Errorf("open config file: %v", err) - } - cfg.DeleteSection("remote \"origin\"") - if err = cfg.SaveToIndent(configPath, "\t"); err != nil { - return fmt.Errorf("save config file: %v", err) - } - return nil -} - // createDelegateHooks creates all the hooks scripts for the repo func createDelegateHooks(repoPath string) (err error) { var ( @@ -1155,30 +905,6 @@ func createDelegateHooks(repoPath string) (err error) { return nil } -// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. -func CleanUpMigrateInfo(repo *Repository) (*Repository, error) { - repoPath := repo.RepoPath() - if err := createDelegateHooks(repoPath); err != nil { - return repo, fmt.Errorf("createDelegateHooks: %v", err) - } - if repo.HasWiki() { - if err := createDelegateHooks(repo.WikiPath()); err != nil { - return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err) - } - } - - if err := cleanUpMigrateGitConfig(repo.GitConfigPath()); err != nil { - return repo, fmt.Errorf("cleanUpMigrateGitConfig: %v", err) - } - if repo.HasWiki() { - if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil { - return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err) - } - } - - return repo, UpdateRepository(repo, false) -} - // initRepoCommit temporarily changes with work directory. func initRepoCommit(tmpPath string, sig *git.Signature) (err error) { var stderr string diff --git a/models/repo_migrate.go b/models/repo_migrate.go new file mode 100644 index 0000000000000..c09c6566df167 --- /dev/null +++ b/models/repo_migrate.go @@ -0,0 +1,289 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "fmt" + "os" + "path" + "strings" + "time" + + "code.gitea.io/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "github.com/Unknwon/com" + ini "gopkg.in/ini.v1" +) + +// MigrateRepoOptions contains the repository migrate options +type MigrateRepoOptions struct { + Name string + Description string + IsPrivate bool + IsMirror bool + RemoteAddr string +} + +/* + GitHub, GitLab, Gogs: *.wiki.git + BitBucket: *.git/wiki +*/ +var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} + +// wikiRemoteURL returns accessible repository URL for wiki if exists. +// Otherwise, it returns an empty string. +func wikiRemoteURL(remote string) string { + remote = strings.TrimSuffix(remote, ".git") + for _, suffix := range commonWikiURLSuffixes { + wikiURL := remote + suffix + if git.IsRepoURLAccessible(wikiURL) { + return wikiURL + } + } + return "" +} + +func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, messageConverter func(error) string) { + repoPath := RepoPath(u.Name, opts.Name) + wikiPath := WikiPath(u.Name, opts.Name) + + failedMigration := func(err error) { + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationFailure, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + Content: messageConverter(err), + }) + // Will no longer delete because the repository is archived + } + + defer func() { + if err := recover(); err != nil { + log.Error(3, "PANIC: Migration failed with panic: %v", err) + + // fail the migration + failedMigration(fmt.Errorf("Migration failed: %v", err)) + } + }() + + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationStarted, + RepoID: repo.ID, + Repo: repo, + Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), + IsPrivate: repo.IsPrivate, + }) + repo.IsArchived = true + if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { + failedMigration(err) + return + } + + if u.IsOrganization() { + t, err := u.GetOwnerTeam() + if err != nil { + failedMigration(err) + return + } + repo.NumWatches = t.NumMembers + } else { + repo.NumWatches = 1 + } + + migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + + if err := os.RemoveAll(repoPath); err != nil { + failedMigration(fmt.Errorf("Failed to remove %s: %v", repoPath, err)) + return + } + + if err := git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + }); err != nil { + failedMigration(fmt.Errorf("Clone: %v", err)) + return + + } + + wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + if len(wikiRemotePath) > 0 { + if err := os.RemoveAll(wikiPath); err != nil { + failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) + return + } + + if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) + return + } + } + } + + // Check if repository is empty. + _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") + if err != nil { + if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { + repo.IsEmpty = true + } else { + failedMigration(fmt.Errorf("check empty: %v - %s", err, stderr)) + return + } + } + + if !repo.IsEmpty { + // Try to get HEAD branch and set it as default branch. + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + failedMigration(fmt.Errorf("OpenRepository: %v", err)) + return + } + headBranch, err := gitRepo.GetHEADBranch() + if err != nil { + failedMigration(fmt.Errorf("GetHEADBranch: %v", err)) + return + } + if headBranch != nil { + repo.DefaultBranch = headBranch.Name + } + + if err = SyncReleasesWithTags(repo, gitRepo); err != nil { + log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) + } + } + + if err = repo.UpdateSize(); err != nil { + log.Error(4, "Failed to update size for repository: %v", err) + } + + if opts.IsMirror { + if _, err = x.InsertOne(&Mirror{ + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + EnablePrune: true, + NextUpdateUnix: util.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + }); err != nil { + failedMigration(fmt.Errorf("InsertOne: %v", err)) + return + } + + repo.IsMirror = true + err = UpdateRepository(repo, false) + } else { + repo, err = CleanUpMigrateInfo(repo) + } + + if err != nil { + if !repo.IsEmpty { + UpdateRepoIndexer(repo) + } + failedMigration(err) + return + } + + repo.IsArchived = false + if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { + failedMigration(err) + return + } + + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionMigrationSuccessful, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), + }) +} + +// MigrateRepository migrates a existing repository from other project hosting. +func MigrateRepository(doer, u *User, opts MigrateRepoOptions, messageConverter func(error) string) (*Repository, error) { + repo, err := CreateRepository(doer, u, CreateRepoOptions{ + Name: opts.Name, + Description: opts.Description, + IsPrivate: opts.IsPrivate, + IsMirror: opts.IsMirror, + NoWatchers: true, + }) + if err != nil { + return nil, err + } + + env, ok := os.LookupEnv("GIT_TERMINAL_PROMPT=0") + os.Setenv("GIT_TERMINAL_PROMPT", "0") + if _, err = git.NewCommand("ls-remote", "-h", opts.RemoteAddr).RunTimeout(1 * time.Minute); err != nil { + if ok { + os.Setenv("GIT_TERMINAL_PROMPT", env) + } else { + os.Unsetenv("GIT_TERMINAL_PROMPT") + } + return repo, fmt.Errorf("Clone: %v", err) + } + if ok { + os.Setenv("GIT_TERMINAL_PROMPT", env) + } else { + os.Unsetenv("GIT_TERMINAL_PROMPT") + } + + // OK if we succeeded above then we know that the clone should start... + go doMigration(doer, u, repo, opts, messageConverter) + + return repo, err +} + +// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". +// This also removes possible user credentials. +func cleanUpMigrateGitConfig(configPath string) error { + cfg, err := ini.Load(configPath) + if err != nil { + return fmt.Errorf("open config file: %v", err) + } + cfg.DeleteSection("remote \"origin\"") + if err = cfg.SaveToIndent(configPath, "\t"); err != nil { + return fmt.Errorf("save config file: %v", err) + } + return nil +} + +// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. +func CleanUpMigrateInfo(repo *Repository) (*Repository, error) { + repoPath := repo.RepoPath() + if err := createDelegateHooks(repoPath); err != nil { + return repo, fmt.Errorf("createDelegateHooks: %v", err) + } + if repo.HasWiki() { + if err := createDelegateHooks(repo.WikiPath()); err != nil { + return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err) + } + } + + if err := cleanUpMigrateGitConfig(repo.GitConfigPath()); err != nil { + return repo, fmt.Errorf("cleanUpMigrateGitConfig: %v", err) + } + if repo.HasWiki() { + if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil { + return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err) + } + } + + return repo, UpdateRepository(repo, false) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5cab6b99d034d..e10e60ca94151 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1716,7 +1716,7 @@ mirror_sync_create = synced new reference %[2]s to %[2]s at %[3]s from mirror migration_started = started a migration from %s to %s migration_successful = successfully migrated from %s to %s -migration_failure = unsuccessfully migrated to %s: %s +migration_failure = unsuccessfully migrated from %s to %s. You should delete this repository. [tool] ago = %s ago diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index 804168cea26ab..038eeb2007095 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -62,7 +62,7 @@ {{else if eq .GetOpType 22}} {{$.i18n.Tr "action.migration_successful" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} {{else if eq .GetOpType 23}} - {{$.i18n.Tr "action.migration_failure" .ShortRepoPath .GetContent | Str2html}} + {{$.i18n.Tr "action.migration_failure" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} {{end}}

{{if or (eq .GetOpType 5) (eq .GetOpType 18)}} From 7b93bd303f1e8964a941312325a2631006b22859 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 4 Feb 2019 21:13:10 +0000 Subject: [PATCH 05/10] Fix Test --- models/release_test.go | 15 ++++++++++++++- models/repo_migrate.go | 7 +++---- routers/api/v1/repo/repo.go | 5 ++++- routers/repo/repo.go | 18 ++++++++++-------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/models/release_test.go b/models/release_test.go index 1b665ffd57d85..3ef4267f23a83 100644 --- a/models/release_test.go +++ b/models/release_test.go @@ -108,11 +108,24 @@ func TestRelease_MirrorDelete(t *testing.T) { IsMirror: true, RemoteAddr: repoPath, } + + out := make(chan bool) + defer close(out) + mirror, err := MigrateRepository(user, user, migrationOptions, func(err error) string { - return err.Error() + if err != nil { + out <- false + return err.Error() + } + out <- true + return repoPath }) assert.NoError(t, err) + // Now we have to wait til the migration is reported. + success := <-out + assert.True(t, success) + gitRepo, err := git.OpenRepository(repoPath) assert.NoError(t, err) diff --git a/models/repo_migrate.go b/models/repo_migrate.go index c09c6566df167..768629fd66d0c 100644 --- a/models/repo_migrate.go +++ b/models/repo_migrate.go @@ -47,7 +47,7 @@ func wikiRemoteURL(remote string) string { return "" } -func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, messageConverter func(error) string) { +func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callback func(error) string) { repoPath := RepoPath(u.Name, opts.Name) wikiPath := WikiPath(u.Name, opts.Name) @@ -59,9 +59,8 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, messa RepoID: repo.ID, Repo: repo, IsPrivate: repo.IsPrivate, - Content: messageConverter(err), + Content: callback(err), }) - // Will no longer delete because the repository is archived } defer func() { @@ -211,7 +210,7 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, messa RepoID: repo.ID, Repo: repo, IsPrivate: repo.IsPrivate, - Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), + Content: callback(nil), }) } diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 084a9a3761b26..cde40172091a7 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -399,7 +399,10 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { IsMirror: form.Mirror, RemoteAddr: remoteAddr, }, func(err error) string { - return util.URLSanitizedError(err, remoteAddr).Error() + if err != nil { + return util.URLSanitizedError(err, remoteAddr).Error() + } + return util.SanitizeURLCredentials(remoteAddr, true) }) if err != nil { err = util.URLSanitizedError(err, remoteAddr) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 1df4ff8e58cbd..963964aead935 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -250,15 +250,17 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { IsMirror: form.Mirror, RemoteAddr: remoteAddr, }, func(err error) string { - err = util.URLSanitizedError(err, remoteAddr) - - if strings.Contains(err.Error(), "Authentication failed") || - strings.Contains(err.Error(), "could not read Username") { - return ctx.Tr("form.auth_failed", err.Error()) - } else if strings.Contains(err.Error(), "fatal:") { - return ctx.Tr("repo.migrate.failed", err.Error()) + if err != nil { + err = util.URLSanitizedError(err, remoteAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return ctx.Tr("form.auth_failed", err.Error()) + } else if strings.Contains(err.Error(), "fatal:") { + return ctx.Tr("repo.migrate.failed", err.Error()) + } + return err.Error() } - return err.Error() + return util.SanitizeURLCredentials(remoteAddr, true) }) if err == nil { log.Trace("Repository migration started [%d]: %s/%s", repo.ID, ctxUser.Name, form.RepoName) From 21c04eeabf4b2dce6a65d71072b6191da0d4ea79 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 5 Feb 2019 20:18:31 +0000 Subject: [PATCH 06/10] Fix failed migration comment --- options/locale/locale_en-US.ini | 2 +- templates/user/dashboard/feeds.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e10e60ca94151..4baa70bea6819 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1716,7 +1716,7 @@ mirror_sync_create = synced new reference %[2]s to %[2]s at %[3]s from mirror migration_started = started a migration from %s to %s migration_successful = successfully migrated from %s to %s -migration_failure = unsuccessfully migrated from %s to %s. You should delete this repository. +migration_failure = unsuccessfully migrated to %s. You should delete this repository. The error was: %s [tool] ago = %s ago diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index 038eeb2007095..b0233620750f2 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -62,7 +62,7 @@ {{else if eq .GetOpType 22}} {{$.i18n.Tr "action.migration_successful" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} {{else if eq .GetOpType 23}} - {{$.i18n.Tr "action.migration_failure" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} + {{$.i18n.Tr "action.migration_failure" .GetRepoLink .ShortRepoPath .GetContent | Str2html}} {{end}}

{{if or (eq .GetOpType 5) (eq .GetOpType 18)}} From a09fae851a6a1b8e829d1f05c32ab1860ac22b20 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 5 Feb 2019 22:29:07 +0000 Subject: [PATCH 07/10] Handle failing and viewing during migration better --- models/repo_migrate.go | 62 +++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/models/repo_migrate.go b/models/repo_migrate.go index 768629fd66d0c..52179d0b34dc9 100644 --- a/models/repo_migrate.go +++ b/models/repo_migrate.go @@ -47,11 +47,30 @@ func wikiRemoteURL(remote string) string { return "" } +func sanitizeRepoPath(path string) string { + return strings.TrimPrefix(path, setting.RepoRootPath) +} + func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callback func(error) string) { repoPath := RepoPath(u.Name, opts.Name) wikiPath := WikiPath(u.Name, opts.Name) + repoPathTmp := repoPath + ".migration" + wikiPathTmp := wikiPath + ".migration" + failedMigration := func(err error) { + if err := os.RemoveAll(wikiPathTmp); err != nil { + log.Error(3, "Failed to remove %s: %v", wikiPathTmp, err) + } + if err := os.RemoveAll(repoPathTmp); err != nil { + log.Error(3, "Failed to remove %s: %v", repoPathTmp, err) + } + + repo.IsEmpty = true + if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { + log.Error(3, "Couldn't set repo to bare:", err) + } + NotifyWatchers(&Action{ ActUserID: doer.ID, ActUser: doer, @@ -81,6 +100,7 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb Content: util.SanitizeURLCredentials(opts.RemoteAddr, true), IsPrivate: repo.IsPrivate, }) + repo.IsEmpty = true repo.IsArchived = true if _, err := x.ID(repo.ID).AllCols().Update(repo); err != nil { failedMigration(err) @@ -100,12 +120,7 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - if err := os.RemoveAll(repoPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", repoPath, err)) - return - } - - if err := git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + if err := git.Clone(opts.RemoteAddr, repoPathTmp, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -115,28 +130,49 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb } + if err := os.RemoveAll(repoPath); err != nil { + log.Error(2, "Migration Failed: unable remove temporary repo %s: %v", repoPath, err) + failedMigration(fmt.Errorf("Failed to remove %s", sanitizeRepoPath(repoPath))) + return + } + + if err := os.Rename(repoPathTmp, repoPath); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", repoPathTmp, repoPath, err) + failedMigration(fmt.Errorf("Failed to rename %s to %s", sanitizeRepoPath(repoPathTmp), sanitizeRepoPath(repoPath))) + return + } + wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) if len(wikiRemotePath) > 0 { - if err := os.RemoveAll(wikiPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) - return - } - - if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + if err := git.Clone(wikiRemotePath, wikiPathTmp, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, Branch: "master", }); err != nil { log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPathTmp); err != nil { + log.Error(2, "Migration Failed: unable remove migrated empty wiki repo %s: %v", wikiPathTmp, err) + failedMigration(fmt.Errorf("Failed to remove migrated empty wiki repo %s", sanitizeRepoPath(wikiPathTmp))) + return + } + } else { if err := os.RemoveAll(wikiPath); err != nil { - failedMigration(fmt.Errorf("Failed to remove %s: %v", wikiPath, err)) + log.Error(2, "Migration Failed: unable remove placeholder repo %s: %v", wikiPath, err) + failedMigration(fmt.Errorf("Failed to remove placeholder repo %s", sanitizeRepoPath(wikiPath))) + return + } + + if err := os.Rename(wikiPathTmp, wikiPath); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) return } } } // Check if repository is empty. + repo.IsEmpty = false _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") if err != nil { if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { From 96cb4daff34da6009c8436a39cfdf5499e4f262e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 6 Feb 2019 20:32:14 +0000 Subject: [PATCH 08/10] Manage deleted repo whilst cloning --- models/repo_migrate.go | 104 +++++++++++++++++++--------- options/locale/locale_en-US.ini | 3 +- templates/user/dashboard/feeds.tmpl | 2 +- 3 files changed, 75 insertions(+), 34 deletions(-) diff --git a/models/repo_migrate.go b/models/repo_migrate.go index 52179d0b34dc9..75ee5ef91de8f 100644 --- a/models/repo_migrate.go +++ b/models/repo_migrate.go @@ -54,16 +54,28 @@ func sanitizeRepoPath(path string) string { func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callback func(error) string) { repoPath := RepoPath(u.Name, opts.Name) wikiPath := WikiPath(u.Name, opts.Name) + timeNow := time.Now().Unix() + repoPathTmp := fmt.Sprintf("%s.migration-%d", repoPath, timeNow) + wikiPathTmp := fmt.Sprintf("%s.migration-%d", wikiPath, timeNow) + repoPathToRemove := fmt.Sprintf("%s.toremove-%d", wikiPath, timeNow) + wikiPathToRemove := fmt.Sprintf("%s.toremove-%d", wikiPath, timeNow) - repoPathTmp := repoPath + ".migration" - wikiPathTmp := wikiPath + ".migration" + repoWorkingPool.CheckIn(com.ToStr(repo.ID)) + defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) failedMigration := func(err error) { - if err := os.RemoveAll(wikiPathTmp); err != nil { - log.Error(3, "Failed to remove %s: %v", wikiPathTmp, err) + RemoveAllWithNotice("Unable to remove temporary wiki path in clean-up for migration", wikiPathTmp) + + RemoveAllWithNotice("Unable to remove temporary repo path in clean-up for migration", repoPathTmp) + + var checkRepo Repository + has, err := x.ID(repo.ID).Get(&checkRepo) + if err != nil { + log.Error(3, "Repository is missing, can't notify", err) } - if err := os.RemoveAll(repoPathTmp); err != nil { - log.Error(3, "Failed to remove %s: %v", repoPathTmp, err) + if !has { + log.Warn("Migration Failed: Target repository is missing, can't notify.") + return } repo.IsEmpty = true @@ -130,50 +142,29 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb } - if err := os.RemoveAll(repoPath); err != nil { - log.Error(2, "Migration Failed: unable remove temporary repo %s: %v", repoPath, err) - failedMigration(fmt.Errorf("Failed to remove %s", sanitizeRepoPath(repoPath))) - return - } - - if err := os.Rename(repoPathTmp, repoPath); err != nil { - log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", repoPathTmp, repoPath, err) - failedMigration(fmt.Errorf("Failed to rename %s to %s", sanitizeRepoPath(repoPathTmp), sanitizeRepoPath(repoPath))) - return - } - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + wikiAvailable := false if len(wikiRemotePath) > 0 { + wikiAvailable = true if err := git.Clone(wikiRemotePath, wikiPathTmp, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, Branch: "master", }); err != nil { + wikiAvailable = false log.Warn("Clone wiki: %v", err) if err := os.RemoveAll(wikiPathTmp); err != nil { log.Error(2, "Migration Failed: unable remove migrated empty wiki repo %s: %v", wikiPathTmp, err) failedMigration(fmt.Errorf("Failed to remove migrated empty wiki repo %s", sanitizeRepoPath(wikiPathTmp))) return } - } else { - if err := os.RemoveAll(wikiPath); err != nil { - log.Error(2, "Migration Failed: unable remove placeholder repo %s: %v", wikiPath, err) - failedMigration(fmt.Errorf("Failed to remove placeholder repo %s", sanitizeRepoPath(wikiPath))) - return - } - - if err := os.Rename(wikiPathTmp, wikiPath); err != nil { - log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) - failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) - return - } } } - // Check if repository is empty. + // Check if repository should be empty. repo.IsEmpty = false - _, stderr, err := com.ExecCmdDir(repoPath, "git", "log", "-1") + _, stderr, err := com.ExecCmdDir(repoPathTmp, "git", "log", "-1") if err != nil { if strings.Contains(stderr, "fatal: bad default revision 'HEAD'") { repo.IsEmpty = true @@ -183,6 +174,46 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb } } + // OK now we're ready to actually begin + // We need to check if the repo still exists + refreshedRepo := Repository{ID: repo.ID} + has, err := x.Get(&refreshedRepo) + if err != nil { + failedMigration(err) + } + if !has { + failedMigration(fmt.Errorf("Clone completed but repository missing")) + } + + if err := os.Rename(repoPath, repoPathToRemove); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + return + } + + if err := os.Rename(repoPathTmp, repoPath); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + return + } + + RemoveAllWithNotice("Unable to remove placeholder repository", wikiPathToRemove) + + if wikiAvailable { + if err := os.Rename(wikiPath, wikiPathToRemove); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + return + } + + if err := os.Rename(wikiPathTmp, wikiPath); err != nil { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + return + } + RemoveAllWithNotice("Unable to remove placeholder wiki repository", wikiPathToRemove) + } + if !repo.IsEmpty { // Try to get HEAD branch and set it as default branch. gitRepo, err := git.OpenRepository(repoPath) @@ -248,6 +279,15 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb IsPrivate: repo.IsPrivate, Content: callback(nil), }) + + NotifyWatchers(&Action{ + ActUserID: doer.ID, + ActUser: doer, + OpType: ActionCreateRepo, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + }) } // MigrateRepository migrates a existing repository from other project hosting. diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d615111b37961..134bf6249804e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1713,10 +1713,11 @@ push_tag = pushed tag %[2]s to %[3]s delete_tag = deleted tag %[2]s from %[3]s delete_branch = deleted branch %[2]s from %[3]s compare_commits = Compare %d commits + mirror_sync_push = synced commits to %[3]s at %[4]s from mirror mirror_sync_create = synced new reference %[2]s to %[3]s from mirror mirror_sync_delete = synced and deleted reference %[2]s at %[3]s from mirror -migration_started = started a migration from %s to %s +migration_started = started a migration from %s to %s. migration_successful = successfully migrated from %s to %s migration_failure = unsuccessfully migrated to %s. You should delete this repository. The error was: %s diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index b0233620750f2..0c1ef02efa35b 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -58,7 +58,7 @@ {{else if eq .GetOpType 20}} {{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} {{else if eq .GetOpType 21}} - {{$.i18n.Tr "action.migration_started" .GetContent .ShortRepoPath | Str2html}} + {{$.i18n.Tr "action.migration_started" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} {{else if eq .GetOpType 22}} {{$.i18n.Tr "action.migration_successful" .GetContent .GetRepoLink .ShortRepoPath | Str2html}} {{else if eq .GetOpType 23}} From 371a272d6f3a8e27a47e6190616b791c72f1240b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 6 Feb 2019 21:46:13 +0000 Subject: [PATCH 09/10] more fixes --- models/release_test.go | 3 +++ models/repo_migrate.go | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/models/release_test.go b/models/release_test.go index 3ef4267f23a83..0344a94e640b4 100644 --- a/models/release_test.go +++ b/models/release_test.go @@ -5,6 +5,7 @@ package models import ( + "log" "testing" "code.gitea.io/git" @@ -120,10 +121,12 @@ func TestRelease_MirrorDelete(t *testing.T) { out <- true return repoPath }) + assert.NoError(t, err) // Now we have to wait til the migration is reported. success := <-out + log.Println("Should have migrated") assert.True(t, success) gitRepo, err := git.OpenRepository(repoPath) diff --git a/models/repo_migrate.go b/models/repo_migrate.go index 75ee5ef91de8f..f57ef85302147 100644 --- a/models/repo_migrate.go +++ b/models/repo_migrate.go @@ -71,10 +71,13 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb var checkRepo Repository has, err := x.ID(repo.ID).Get(&checkRepo) if err != nil { - log.Error(3, "Repository is missing, can't notify", err) + log.Error(3, "Migration Failed: Repository is missing, can't notify: %v", err) + callback(fmt.Errorf("Migration Failed: Target repository is missing, can't notify: %v", err)) + return } if !has { log.Warn("Migration Failed: Target repository is missing, can't notify.") + callback(fmt.Errorf("Migration Failed: Target repository is missing, can't notify")) return } @@ -185,24 +188,24 @@ func doMigration(doer, u *User, repo *Repository, opts MigrateRepoOptions, callb failedMigration(fmt.Errorf("Clone completed but repository missing")) } - if err := os.Rename(repoPath, repoPathToRemove); err != nil { - log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) - failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + if err := os.Rename(repoPath, repoPathToRemove); err != nil && !os.IsNotExist(err) { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", repoPath, repoPathToRemove, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(repoPath), sanitizeRepoPath(repoPathToRemove))) return } if err := os.Rename(repoPathTmp, repoPath); err != nil { - log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) - failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", repoPathTmp, repoPath, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(repoPathTmp), sanitizeRepoPath(repoPath))) return } - RemoveAllWithNotice("Unable to remove placeholder repository", wikiPathToRemove) + RemoveAllWithNotice("Unable to remove placeholder repository", repoPathToRemove) if wikiAvailable { - if err := os.Rename(wikiPath, wikiPathToRemove); err != nil { - log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPathTmp, wikiPath, err) - failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPathTmp), sanitizeRepoPath(wikiPath))) + if err := os.Rename(wikiPath, wikiPathToRemove); err != nil && !os.IsNotExist(err) { + log.Error(2, "Migration Failed: unable rename temporary migrated repo %s to %s: %v", wikiPath, wikiPathToRemove, err) + failedMigration(fmt.Errorf("Failed to rename temporary migrated repo %s to %s", sanitizeRepoPath(wikiPath), sanitizeRepoPath(wikiPathToRemove))) return } From 38b6ae099b106419b225f45e7524a940905e56ba Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 7 Feb 2019 16:12:55 +0000 Subject: [PATCH 10/10] Remove extraneous println Signed-off-by: Andrew Thornton --- models/release_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/release_test.go b/models/release_test.go index 0344a94e640b4..45e4741b75420 100644 --- a/models/release_test.go +++ b/models/release_test.go @@ -5,7 +5,6 @@ package models import ( - "log" "testing" "code.gitea.io/git" @@ -126,7 +125,6 @@ func TestRelease_MirrorDelete(t *testing.T) { // Now we have to wait til the migration is reported. success := <-out - log.Println("Should have migrated") assert.True(t, success) gitRepo, err := git.OpenRepository(repoPath)