Add LFS data mirroring

integrations/lfs_getobject_test.go

@@ -16,7 +16,6 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/routers/routes"
@@ -50,7 +49,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
 	lfsID++
 	lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
 	assert.NoError(t, err)
-	contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
+	contentStore := &models.ContentStore{ObjectStorage: storage.LFS}
 	exist, err := contentStore.Exists(lfsMetaObject)
 	assert.NoError(t, err)
 	if !exist {

models/error.go

@@ -56,6 +56,20 @@ func (err ErrNamePatternNotAllowed) Error() string {
 	return fmt.Sprintf("name pattern is not allowed [pattern: %s]", err.Pattern)
 }
 
+// ErrMirrorLFSServerNotValid represents an "LFS Server not valid" error.
+type ErrMirrorLFSServerNotValid struct {
+}
+
+// IsMirrorLFSServerValid checks if an error is an ErrMirrorLFSServerNotValid.
+func IsMirrorLFSServerValid(err error) bool {
+	_, ok := err.(ErrMirrorLFSServerNotValid)
+	return ok
+}
+
+func (err ErrMirrorLFSServerNotValid) Error() string {
+	return "LFS Server not valid"
+}
+
 // ErrNameCharsNotAllowed represents a "character not allowed in name" error.
 type ErrNameCharsNotAllowed struct {
 	Name string

models/lfs.go

@@ -11,7 +11,12 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"strconv"
+	"strings"
+	"time"
 
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/timeutil"
 
 	"xorm.io/builder"
@@ -27,6 +32,70 @@ type LFSMetaObject struct {
 	CreatedUnix  timeutil.TimeStamp `xorm:"created"`
 }
 
+// LFSMetaObjectBasic represents basic LFS metadata.
+type LFSMetaObjectBasic struct {
+	Oid  string `json:"oid"`
+	Size int64  `json:"size"`
+}
+
+// IsPointerFileAndStored will return a partially filled LFSMetaObject if the provided byte slice is a pointer file and stored in contentStore
+func IsPointerFileAndStored(buf *[]byte) *LFSMetaObject {
+	if !setting.LFS.StartServer {
+		return nil
+	}
+
+	headString := string(*buf)
+	if !strings.HasPrefix(headString, LFSMetaFileIdentifier) {
+		return nil
+	}
+
+	splitLines := strings.Split(headString, "\n")
+	if len(splitLines) < 3 {
+		return nil
+	}
+
+	oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix)
+	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+	if len(oid) != 64 || err != nil {
+		return nil
+	}
+
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
+	meta := &LFSMetaObject{Oid: oid, Size: size}
+	exist, err := contentStore.Exists(meta)
+	if err != nil || !exist {
+		return nil
+	}
+
+	return meta
+}
+
+// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
+func IsPointerFile(buf *[]byte) *LFSMetaObjectBasic {
+	if !setting.LFS.StartServer {
+		return nil
+	}
+
+	headString := string(*buf)
+	if !strings.HasPrefix(headString, LFSMetaFileIdentifier) {
+		return nil
+	}
+
+	splitLines := strings.Split(headString, "\n")
+	if len(splitLines) < 3 {
+		return nil
+	}
+
+	oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix)
+	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+	if len(oid) != 64 || err != nil {
+		return nil
+	}
+	meta := &LFSMetaObjectBasic{Oid: oid, Size: size}
+
+	return meta
+}
+
 // RelativePath returns the relative path of the lfs object
 func (m *LFSMetaObject) RelativePath() string {
 	if len(m.Oid) < 5 {
@@ -234,3 +303,31 @@ func IterateLFS(f func(mo *LFSMetaObject) error) error {
 		}
 	}
 }
+
+// BatchResponse contains multiple object metadata Representation structures
+// for use with the batch API.
+type BatchResponse struct {
+	Transfer string            `json:"transfer,omitempty"`
+	Objects  []*Representation `json:"objects"`
+}
+
+// Representation is object metadata as seen by clients of the lfs server.
+type Representation struct {
+	Oid     string           `json:"oid"`
+	Size    int64            `json:"size"`
+	Actions map[string]*Link `json:"actions"`
+	Error   *ObjectError     `json:"error,omitempty"`
+}
+
+// ObjectError defines the JSON structure returned to the client in case of an error
+type ObjectError struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+// Link provides a structure used to build a hypermedia representation of an HTTP link.
+type Link struct {
+	Href      string            `json:"href"`
+	Header    map[string]string `json:"header,omitempty"`
+	ExpiresAt time.Time         `json:"expires_at,omitempty"`
+}

modules/lfs/content_store.go → models/lfs_content_store.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package lfs
+package models
 
 import (
 	"crypto/sha256"
@@ -13,7 +13,6 @@ import (
 	"io"
 	"os"
 
-	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
 )
@@ -45,7 +44,7 @@ type ContentStore struct {
 
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.Reader. If fromByte > 0, the reader starts from that byte
-func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
+func (s *ContentStore) Get(meta *LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
 	f, err := s.Open(meta.RelativePath())
 	if err != nil {
 		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
@@ -66,7 +65,7 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 }
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
-func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
+func (s *ContentStore) Put(meta *LFSMetaObject, r io.Reader) error {
 	p := meta.RelativePath()
 
 	// Wrap the provided reader with an inline hashing and size checker
@@ -92,7 +91,7 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
+func (s *ContentStore) Exists(meta *LFSMetaObject) (bool, error) {
 	_, err := s.ObjectStorage.Stat(meta.RelativePath())
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -104,7 +103,7 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
-func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
+func (s *ContentStore) Verify(meta *LFSMetaObject) (bool, error) {
 	p := meta.RelativePath()
 	fi, err := s.ObjectStorage.Stat(p)
 	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {

models/repo.go

@@ -215,10 +215,12 @@ type Repository struct {
 	NumClosedProjects   int `xorm:"NOT NULL DEFAULT 0"`
 	NumOpenProjects     int `xorm:"-"`
 
-	IsPrivate  bool `xorm:"INDEX"`
-	IsEmpty    bool `xorm:"INDEX"`
-	IsArchived bool `xorm:"INDEX"`
-	IsMirror   bool `xorm:"INDEX"`
+	IsPrivate  bool   `xorm:"INDEX"`
+	IsEmpty    bool   `xorm:"INDEX"`
+	IsArchived bool   `xorm:"INDEX"`
+	IsMirror   bool   `xorm:"INDEX"`
+	LFS        bool   `xorm:"INDEX"`
+	LFSServer  string `xorm:"TEXT"`
 	*Mirror    `xorm:"-"`
 	Status     RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
 
@@ -945,7 +947,7 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
 }
 
 // CheckCreateRepository check if could created a repository
-func CheckCreateRepository(doer, u *User, name string, overwriteOrAdopt bool) error {
+func CheckCreateRepository(doer, u *User, name string, lfs bool, lfsServer string, overwriteOrAdopt bool) error {
 	if !doer.CanCreateRepo() {
 		return ErrReachLimitOfRepo{u.MaxRepoCreation}
 	}
@@ -969,6 +971,13 @@ func CheckCreateRepository(doer, u *User, name string, overwriteOrAdopt bool) er
 	if !overwriteOrAdopt && isExist {
 		return ErrRepoFilesAlreadyExist{u.Name, name}
 	}
+
+	if lfs {
+		_, err := url.ParseRequestURI(lfsServer)
+		if err != nil {
+			return ErrMirrorLFSServerNotValid{}
+		}
+	}
 	return nil
 }
 

modules/forms/repo_form.go

@@ -73,6 +73,9 @@ type MigrateRepoForm struct {
 	// required: true
 	RepoName       string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
 	Mirror         bool   `json:"mirror"`
+	LFS            bool   `json:"lfs"`
+	LFSServer      string `json:"lfs_server"`
+	LFSFetchOlder  bool   `json:"lfs_fetch_older"`
 	Private        bool   `json:"private"`
 	Description    string `json:"description" binding:"MaxSize(255)"`
 	Wiki           bool   `json:"wiki"`

modules/git/blob.go

@@ -32,6 +32,19 @@ func (b *Blob) GetBlobContent() (string, error) {
 	return string(buf), nil
 }
 
+// GetBlobFirstBytes gets limited content of the blob as bytes
+func (b *Blob) GetBlobFirstBytes(limit int) ([]byte, error) {
+	dataRc, err := b.DataAsync()
+	buf := make([]byte, limit)
+	if err != nil {
+		return buf, err
+	}
+	defer dataRc.Close()
+	n, _ := dataRc.Read(buf)
+	buf = buf[:n]
+	return buf, nil
+}
+
 // GetBlobLineCount gets line count of lob as raw text
 func (b *Blob) GetBlobLineCount() (int, error) {
 	reader, err := b.DataAsync()

modules/lfs/pointers.go

@@ -6,8 +6,6 @@ package lfs
 
 import (
 	"io"
-	"strconv"
-	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
@@ -29,43 +27,11 @@ func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) {
 		return nil, nil
 	}
 
-	return IsPointerFile(&buf), &buf
-}
-
-// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
-func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
-	if !setting.LFS.StartServer {
-		return nil
-	}
-
-	headString := string(*buf)
-	if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
-		return nil
-	}
-
-	splitLines := strings.Split(headString, "\n")
-	if len(splitLines) < 3 {
-		return nil
-	}
-
-	oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
-	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
-	if len(oid) != 64 || err != nil {
-		return nil
-	}
-
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
-	meta := &models.LFSMetaObject{Oid: oid, Size: size}
-	exist, err := contentStore.Exists(meta)
-	if err != nil || !exist {
-		return nil
-	}
-
-	return meta
+	return models.IsPointerFileAndStored(&buf), &buf
 }
 
 // ReadMetaObject will read a models.LFSMetaObject and return a reader
 func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
-	contentStore := &ContentStore{ObjectStorage: storage.LFS}
+	contentStore := &models.ContentStore{ObjectStorage: storage.LFS}
 	return contentStore.Get(meta, 0)
 }