Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add LFS data mirroring #14718

Closed
wants to merge 17 commits into from
Closed

Add LFS data mirroring #14718

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
36582e7
Add LFS data mirroring
bl00mber Feb 17, 2021
e7941de
amend for gogs
bl00mber Feb 18, 2021
3f02813
Merge branch 'master' into lfs-data-mirroring
bl00mber Feb 18, 2021
a727324
add "git-lfs not installed" error
bl00mber Feb 19, 2021
64370e0
Merge branch 'master' into lfs-data-mirroring
bl00mber Feb 19, 2021
3b99147
fetch lfs to gitea setting.LFS.Path/tmp
bl00mber Feb 19, 2021
7c744c1
add size restrictions
bl00mber Feb 19, 2021
cd1dfbb
Rewritten on LFS API
bl00mber Feb 28, 2021
61599e8
Fix circular dependencies
bl00mber Mar 1, 2021
4c649db
add fetchingMetaObjectsSet
bl00mber Mar 1, 2021
d35bfcc
fetch LFS on mirror update
bl00mber Mar 1, 2021
183c6d2
do not fetch existing files with the same size & lint files
bl00mber Mar 1, 2021
5677bcc
add error ErrMirrorLFSServerNotValid
bl00mber Mar 1, 2021
9982edb
update
bl00mber Mar 1, 2021
5550e2e
Merge branch 'master' into lfs-data-mirroring
bl00mber Mar 1, 2021
670a4e7
update
bl00mber Mar 1, 2021
1524300
Merge branch 'master' of github.com:go-gitea/gitea into lfs-data-mirr…
bl00mber Mar 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions modules/forms/repo_form.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ type MigrateRepoForm struct {
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Expand Down
1 change: 1 addition & 0 deletions modules/migrations/base/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type MigrateOptions struct {
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
Expand Down
1 change: 1 addition & 0 deletions modules/migrations/gitea_uploader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
LFS: opts.LFS,
CloneAddr: repo.CloneURL,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Expand Down
71 changes: 71 additions & 0 deletions modules/repository/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ package repository
import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -70,6 +72,75 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
return repo, fmt.Errorf("Clone: %v", err)
}

if opts.LFS {
bl00mber marked this conversation as resolved.
Show resolved Hide resolved
_, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath)
bl00mber marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err)
}

lfsSrc := path.Join(repoPath, "lfs", "objects")
lfsDst := path.Join(setting.LFS.Path)

// move LFS files
err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error {
var relSrcPath = strings.Replace(path, lfsSrc, "", 1)
if relSrcPath == "" {
return nil
}
if err != nil {
return err
}
lfsSrcFull := filepath.Join(lfsSrc, relSrcPath)
lfsDstFull := filepath.Join(lfsDst, relSrcPath)

if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) {
return nil
}

if info.IsDir() {
return os.Mkdir(lfsDstFull, 0755)
}

// generate and associate LFS OIDs
file, err := os.Open(lfsSrcFull)
if err != nil {
return err
}
defer file.Close()

oid, err := models.GenerateLFSOid(file)
if err != nil {
return err
}
fileInfo, err := file.Stat()
if err != nil {
return err
}

lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:])
err = os.Rename(lfsSrcFull, lfsDstFull)
if err != nil {
return err
}

_, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID})
if err != nil {
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err)
return err
}

return nil
})
if err != nil {
return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err)
}

err = os.RemoveAll(path.Join(repoPath, "lfs"))
if err != nil {
return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err)
}
}

if opts.Wiki {
wikiPath := models.WikiPath(u.Name, opts.RepoName)
wikiRemotePath := WikiRemoteURL(opts.CloneAddr)
Expand Down
1 change: 1 addition & 0 deletions modules/structs/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ type MigrateRepoOptions struct {
AuthToken string `json:"auth_token"`

Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Expand Down
4 changes: 2 additions & 2 deletions options/locale/locale_en-US.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,7 @@ need_auth = Clone Authorization
migrate_options = Migration Options
migrate_service = Migration Service
migrate_options_mirror_helper = This repository will be a <span class="text blue">mirror</span>
migrate_options_mirror_lfs = Mirror LFS data
migrate_options_mirror_disabled = Your site administrator has disabled new mirrors.
migrate_items = Migration Items
migrate_items_wiki = Wiki
Expand All @@ -770,7 +771,6 @@ migrate.clone_local_path = or a local server path
migrate.permission_denied = You are not allowed to import local repositories.
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
migrate.failed = Migration failed: %v
migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead.
migrate.migrate_items_options = Access Token is required to migrate additional items
migrated_from = Migrated from <a href="%[1]s">%[2]s</a>
migrated_from_fake = Migrated From %[1]s
Expand Down Expand Up @@ -931,7 +931,7 @@ ext_issues = Ext. Issues
ext_issues.desc = Link to an external issue tracker.

projects = Projects
projects.desc = Manage issues and pulls in project boards.
projects.desc = Manage issues and pulls in project boards.
projects.description = Description (optional)
projects.description_placeholder = Description
projects.create = Create Project
Expand Down
1 change: 1 addition & 0 deletions routers/api/v1/repo/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func Migrate(ctx *context.APIContext) {
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror,
LFS: form.LFS,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Expand Down
2 changes: 2 additions & 0 deletions routers/repo/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func Migrate(ctx *context.Context) {
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
ctx.Data["mirror"] = ctx.Query("mirror") == "1"
ctx.Data["LFS"] = ctx.Query("lfs") == "1"
ctx.Data["wiki"] = ctx.Query("wiki") == "1"
ctx.Data["milestones"] = ctx.Query("milestones") == "1"
ctx.Data["labels"] = ctx.Query("labels") == "1"
Expand Down Expand Up @@ -172,6 +173,7 @@ func MigratePost(ctx *context.Context) {
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
LFS: form.LFS,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/git.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
Expand All @@ -30,15 +29,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<div class="ui divider"></div>
Expand Down
13 changes: 9 additions & 4 deletions templates/repo/migrate/gitea.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/github.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
15 changes: 10 additions & 5 deletions templates/repo/migrate/gitlab.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
13 changes: 9 additions & 4 deletions templates/repo/migrate/gogs.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>

Expand All @@ -27,15 +26,21 @@

<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
{{if .DisableMirrors}}
<div class="ui checkbox">
{{if .DisableMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}
</div>
{{else}}
<div class="ui checkbox">
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
{{end}}
</div>
<div class="ui checkbox">
6543 marked this conversation as resolved.
Show resolved Hide resolved
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
</div>
{{end}}
</div>

<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
Expand Down
8 changes: 8 additions & 0 deletions templates/swagger/v1_json.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14636,6 +14636,10 @@
"type": "boolean",
"x-go-name": "Labels"
},
"lfs": {
"type": "boolean",
"x-go-name": "LFS"
},
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
Expand Down Expand Up @@ -14715,6 +14719,10 @@
"type": "boolean",
"x-go-name": "Labels"
},
"lfs": {
"type": "boolean",
"x-go-name": "LFS"
},
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
Expand Down