Allow adding new files to an empty repo (#24164)

![image](https://user-images.githubusercontent.com/2114189/232561612-2bfcfd0a-fc04-47ba-965f-5d0bcea46c54.png)

models/db/iterate_test.go

@@ -19,13 +19,16 @@ func TestIterate(t *testing.T) {
 	xe := unittest.GetXORMEngine()
 	assert.NoError(t, xe.Sync(&repo_model.RepoUnit{}))
 
-	var repoCnt int
-	err := db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repo *repo_model.RepoUnit) error {
-		repoCnt++
+	cnt, err := db.GetEngine(db.DefaultContext).Count(&repo_model.RepoUnit{})
+	assert.NoError(t, err)
+
+	var repoUnitCnt int
+	err = db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repo *repo_model.RepoUnit) error {
+		repoUnitCnt++
 		return nil
 	})
 	assert.NoError(t, err)
-	assert.EqualValues(t, 89, repoCnt)
+	assert.EqualValues(t, cnt, repoUnitCnt)
 
 	err = db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repoUnit *repo_model.RepoUnit) error {
 		reopUnit2 := repo_model.RepoUnit{ID: repoUnit.ID}

models/db/list_test.go

@@ -31,15 +31,20 @@ func TestFind(t *testing.T) {
 	xe := unittest.GetXORMEngine()
 	assert.NoError(t, xe.Sync(&repo_model.RepoUnit{}))
 
+	var repoUnitCount int
+	_, err := db.GetEngine(db.DefaultContext).SQL("SELECT COUNT(*) FROM repo_unit").Get(&repoUnitCount)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, repoUnitCount)
+
 	opts := mockListOptions{}
 	var repoUnits []repo_model.RepoUnit
-	err := db.Find(db.DefaultContext, &opts, &repoUnits)
+	err = db.Find(db.DefaultContext, &opts, &repoUnits)
 	assert.NoError(t, err)
-	assert.EqualValues(t, 89, len(repoUnits))
+	assert.EqualValues(t, repoUnitCount, len(repoUnits))
 
 	cnt, err := db.Count(db.DefaultContext, &opts, new(repo_model.RepoUnit))
 	assert.NoError(t, err)
-	assert.EqualValues(t, 89, cnt)
+	assert.EqualValues(t, repoUnitCount, cnt)
 
 	repoUnits = make([]repo_model.RepoUnit, 0, 10)
 	newCnt, err := db.FindAndCount(db.DefaultContext, &opts, &repoUnits)

models/dbfs/dbfs_test.go

@@ -12,8 +12,6 @@ import (
 	"code.gitea.io/gitea/models/db"
 
 	"github.com/stretchr/testify/assert"
-
-	_ "github.com/mattn/go-sqlite3"
 )
 
 func changeDefaultFileBlockSize(n int64) (restore func()) {

models/fixtures/repo_unit.yml

@@ -601,3 +601,9 @@
   repo_id: 57
   type: 5
   created_unix: 946684810
+
+-
+  id: 90
+  repo_id: 52
+  type: 1
+  created_unix: 946684810

models/fixtures/repository.yml

@@ -1560,6 +1560,7 @@
   owner_name: user30
   lower_name: empty
   name: empty
+  default_branch: master
   num_watches: 0
   num_stars: 0
   num_forks: 0

models/fixtures/user.yml

@@ -1091,7 +1091,7 @@
   max_repo_creation: -1
   is_active: true
   is_admin: false
-  is_restricted: true
+  is_restricted: false
   allow_git_hook: false
   allow_import_local: false
   allow_create_organization: true

models/repo/repo.go

@@ -225,6 +225,12 @@ func (repo *Repository) IsBroken() bool {
 	return repo.Status == RepositoryBroken
 }
 
+// MarkAsBrokenEmpty marks the repo as broken and empty
+func (repo *Repository) MarkAsBrokenEmpty() {
+	repo.Status = RepositoryBroken
+	repo.IsEmpty = true
+}
+
 // AfterLoad is invoked from XORM after setting the values of all fields of this object.
 func (repo *Repository) AfterLoad() {
 	repo.NumOpenIssues = repo.NumIssues - repo.NumClosedIssues
@@ -729,7 +735,7 @@ func IsRepositoryExist(ctx context.Context, u *user_model.User, repoName string)
 		return false, err
 	}
 	isDir, err := util.IsDir(RepoPath(u.Name, repoName))
-	return has && isDir, err
+	return has || isDir, err
 }
 
 // GetTemplateRepo populates repo.TemplateRepo for a generated repository and

models/unittest/fixtures.go

@@ -17,7 +17,7 @@ import (
 	"xorm.io/xorm/schemas"
 )
 
-var fixtures *testfixtures.Loader
+var fixturesLoader *testfixtures.Loader
 
 // GetXORMEngine gets the XORM engine
 func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
@@ -30,11 +30,11 @@ func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) {
 // InitFixtures initialize test fixtures for a test database
 func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	e := GetXORMEngine(engine...)
-	var testfiles func(*testfixtures.Loader) error
+	var fixtureOptionFiles func(*testfixtures.Loader) error
 	if opts.Dir != "" {
-		testfiles = testfixtures.Directory(opts.Dir)
+		fixtureOptionFiles = testfixtures.Directory(opts.Dir)
 	} else {
-		testfiles = testfixtures.Files(opts.Files...)
+		fixtureOptionFiles = testfixtures.Files(opts.Files...)
 	}
 	dialect := "unknown"
 	switch e.Dialect().URI().DBType {
@@ -54,14 +54,14 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 		testfixtures.Database(e.DB().DB),
 		testfixtures.Dialect(dialect),
 		testfixtures.DangerousSkipTestDatabaseCheck(),
-		testfiles,
+		fixtureOptionFiles,
 	}
 
 	if e.Dialect().URI().DBType == schemas.POSTGRES {
 		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
 	}
 
-	fixtures, err = testfixtures.New(loaderOptions...)
+	fixturesLoader, err = testfixtures.New(loaderOptions...)
 	if err != nil {
 		return err
 	}
@@ -78,11 +78,9 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 func LoadFixtures(engine ...*xorm.Engine) error {
 	e := GetXORMEngine(engine...)
 	var err error
-	// Database transaction conflicts could occur and result in ROLLBACK
-	// As a simple workaround, we just retry 20 times.
-	for i := 0; i < 20; i++ {
-		err = fixtures.Load()
-		if err == nil {
+	// (doubt) database transaction conflicts could occur and result in ROLLBACK? just try for a few times.
+	for i := 0; i < 5; i++ {
+		if err = fixturesLoader.Load(); err == nil {
 			break
 		}
 		time.Sleep(200 * time.Millisecond)

models/user/user_test.go

@@ -5,6 +5,7 @@ package user_test
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
 	"strings"
 	"testing"
@@ -64,9 +65,10 @@ func TestSearchUsers(t *testing.T) {
 	testSuccess := func(opts *user_model.SearchUserOptions, expectedUserOrOrgIDs []int64) {
 		users, _, err := user_model.SearchUsers(opts)
 		assert.NoError(t, err)
-		if assert.Len(t, users, len(expectedUserOrOrgIDs), opts) {
+		cassText := fmt.Sprintf("ids: %v, opts: %v", expectedUserOrOrgIDs, opts)
+		if assert.Len(t, users, len(expectedUserOrOrgIDs), "case: %s", cassText) {
 			for i, expectedID := range expectedUserOrOrgIDs {
-				assert.EqualValues(t, expectedID, users[i].ID)
+				assert.EqualValues(t, expectedID, users[i].ID, "case: %s", cassText)
 			}
 		}
 	}
@@ -118,7 +120,7 @@ func TestSearchUsers(t *testing.T) {
 		[]int64{1})
 
 	testUserSuccess(&user_model.SearchUserOptions{ListOptions: db.ListOptions{Page: 1}, IsRestricted: util.OptionalBoolTrue},
-		[]int64{29, 30})
+		[]int64{29})
 
 	testUserSuccess(&user_model.SearchUserOptions{ListOptions: db.ListOptions{Page: 1}, IsProhibitLogin: util.OptionalBoolTrue},
 		[]int64{30})

modules/context/context.go

@@ -301,7 +301,7 @@ func (ctx *Context) serverErrorInternal(logMsg string, logErr error) {
 
 		// it's safe to show internal error to admin users, and it helps
 		if !setting.IsProd || (ctx.Doer != nil && ctx.Doer.IsAdmin) {
-			ctx.Data["ErrorMsg"] = logErr
+			ctx.Data["ErrorMsg"] = fmt.Sprintf("%s, %s", logMsg, logErr)
 		}
 	}
 

modules/context/repo.go

@@ -184,6 +184,9 @@ func (r *Repository) CanCreateIssueDependencies(user *user_model.User, isPull bo
 
 // GetCommitsCount returns cached commit count for current view
 func (r *Repository) GetCommitsCount() (int64, error) {
+	if r.Commit == nil {
+		return 0, nil
+	}
 	var contextName string
 	if r.IsViewBranch {
 		contextName = r.BranchName
@@ -642,8 +645,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	if err != nil {
 		if strings.Contains(err.Error(), "repository does not exist") || strings.Contains(err.Error(), "no such file or directory") {
 			log.Error("Repository %-v has a broken repository on the file system: %s Error: %v", ctx.Repo.Repository, ctx.Repo.Repository.RepoPath(), err)
-			ctx.Repo.Repository.Status = repo_model.RepositoryBroken
-			ctx.Repo.Repository.IsEmpty = true
+			ctx.Repo.Repository.MarkAsBrokenEmpty()
 			ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch
 			// Only allow access to base of repo or settings
 			if !isHomeOrSettings {
@@ -689,7 +691,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	ctx.Data["BranchesCount"] = len(brs)
 
 	// If not branch selected, try default one.
-	// If default branch doesn't exists, fall back to some other branch.
+	// If default branch doesn't exist, fall back to some other branch.
 	if len(ctx.Repo.BranchName) == 0 {
 		if len(ctx.Repo.Repository.DefaultBranch) > 0 && gitRepo.IsBranchExist(ctx.Repo.Repository.DefaultBranch) {
 			ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch
@@ -878,6 +880,10 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 	return func(ctx *Context) (cancel context.CancelFunc) {
 		// Empty repository does not have reference information.
 		if ctx.Repo.Repository.IsEmpty {
+			// assume the user is viewing the (non-existent) default branch
+			ctx.Repo.IsViewBranch = true
+			ctx.Repo.BranchName = ctx.Repo.Repository.DefaultBranch
+			ctx.Data["TreePath"] = ""
 			return
 		}
 
@@ -907,27 +913,30 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 			refName = ctx.Repo.Repository.DefaultBranch
 			if !ctx.Repo.GitRepo.IsBranchExist(refName) {
 				brs, _, err := ctx.Repo.GitRepo.GetBranchNames(0, 0)
-				if err != nil {
-					ctx.ServerError("GetBranches", err)
-					return
+				if err == nil && len(brs) != 0 {
+					refName = brs[0]
 				} else if len(brs) == 0 {
-					err = fmt.Errorf("No branches in non-empty repository %s",
-						ctx.Repo.GitRepo.Path)
-					ctx.ServerError("GetBranches", err)
-					return
+					log.Error("No branches in non-empty repository %s", ctx.Repo.GitRepo.Path)
+					ctx.Repo.Repository.MarkAsBrokenEmpty()
+				} else {
+					log.Error("GetBranches error: %v", err)
+					ctx.Repo.Repository.MarkAsBrokenEmpty()
 				}
-				refName = brs[0]
 			}
 			ctx.Repo.RefName = refName
 			ctx.Repo.BranchName = refName
 			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName)
-			if err != nil {
+			if err == nil {
+				ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
+			} else if strings.Contains(err.Error(), "fatal: not a git repository") || strings.Contains(err.Error(), "object does not exist") {
+				// if the repository is broken, we can continue to the handler code, to show "Settings -> Delete Repository" for end users
+				log.Error("GetBranchCommit: %v", err)
+				ctx.Repo.Repository.MarkAsBrokenEmpty()
+			} else {
 				ctx.ServerError("GetBranchCommit", err)
 				return
 			}
-			ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
 			ctx.Repo.IsViewBranch = true
-
 		} else {
 			refName = getRefName(ctx, refType)
 			ctx.Repo.RefName = refName

modules/git/command.go

@@ -211,10 +211,18 @@ type RunOpts struct {
 	Env               []string
 	Timeout           time.Duration
 	UseContextTimeout bool
-	Dir               string
-	Stdout, Stderr    io.Writer
-	Stdin             io.Reader
-	PipelineFunc      func(context.Context, context.CancelFunc) error
+
+	// Dir is the working dir for the git command, however:
+	// FIXME: this could be incorrect in many cases, for example:
+	// * /some/path/.git
+	// * /some/path/.git/gitea-data/data/repositories/user/repo.git
+	// If "user/repo.git" is invalid/broken, then running git command in it will use "/some/path/.git", and produce unexpected results
+	// The correct approach is to use `--git-dir" global argument
+	Dir string
+
+	Stdout, Stderr io.Writer
+	Stdin          io.Reader
+	PipelineFunc   func(context.Context, context.CancelFunc) error
 }
 
 func commonBaseEnvs() []string {

modules/git/repo.go

@@ -80,7 +80,7 @@ func InitRepository(ctx context.Context, repoPath string, bare bool) error {
 // IsEmpty Check if repository is empty.
 func (repo *Repository) IsEmpty() (bool, error) {
 	var errbuf, output strings.Builder
-	if err := NewCommand(repo.Ctx, "show-ref", "--head", "^HEAD$").
+	if err := NewCommand(repo.Ctx).AddOptionFormat("--git-dir=%s", repo.Path).AddArguments("show-ref", "--head", "^HEAD$").
 		Run(&RunOpts{
 			Dir:    repo.Path,
 			Stdout: &output,

modules/git/repo_base_nogogit.go

@@ -61,7 +61,7 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) {
 	}
 
 	repo.batchWriter, repo.batchReader, repo.batchCancel = CatFileBatch(ctx, repoPath)
-	repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(ctx, repo.Path)
+	repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(ctx, repoPath)
 
 	return repo, nil
 }

modules/indexer/code/indexer.go

@@ -154,7 +154,9 @@ func Init() {
 				log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
 
 				if err := index(ctx, indexer, indexerData.RepoID); err != nil {
-					log.Error("index: %v", err)
+					if !setting.IsInTesting {
+						log.Error("indexer index error for repo %v: %v", indexerData.RepoID, err)
+					}
 					if indexer.Ping() {
 						continue
 					}

modules/indexer/stats/db.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // DBIndexer implements Indexer interface to use database's like search
@@ -46,7 +47,7 @@ func (db *DBIndexer) Index(id int64) error {
 	// Get latest commit for default branch
 	commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
 	if err != nil {
-		if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) {
+		if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) || setting.IsInTesting {
 			log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath())
 			return nil
 		}
@@ -62,7 +63,9 @@ func (db *DBIndexer) Index(id int64) error {
 	// Calculate and save language statistics to database
 	stats, err := gitRepo.GetLanguageStats(commitID)
 	if err != nil {
-		log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
+		if !setting.IsInTesting {
+			log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
+		}
 		return err
 	}
 	err = repo_model.UpdateLanguageStats(repo, commitID, stats)

modules/indexer/stats/queue.go

@@ -10,6 +10,7 @@ import (
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/queue"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // statsQueue represents a queue to handle repository stats updates
@@ -20,7 +21,9 @@ func handle(data ...queue.Data) []queue.Data {
 	for _, datum := range data {
 		opts := datum.(int64)
 		if err := indexer.Index(opts); err != nil {
-			log.Error("stats queue indexer.Index(%d) failed: %v", opts, err)
+			if !setting.IsInTesting {
+				log.Error("stats queue indexer.Index(%d) failed: %v", opts, err)
+			}
 		}
 	}
 	return nil