diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b90e38812698..f3170405a3be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,50 @@ This changelog goes through all the changes that have been made in each release without substantial changes to our git log; to see the highlights of what has been added to each release, please refer to the [blog](https://blog.gitea.io). +## [1.17.2](https://github.com/go-gitea/gitea/releases/tag/v1.17.2) - 2022-09-06 + +* SECURITY + * Double check CloneURL is acceptable (#20869) (#20892) + * Add more checks in migration code (#21011) (#21050) +* ENHANCEMENTS + * Fix hard-coded timeout and error panic in API archive download endpoint (#20925) (#21051) + * Improve arc-green code theme (#21039) (#21042) + * Enable contenthash in filename for dynamic assets (#20813) (#20932) + * Don't open new page for ext wiki on same repository (#20725) (#20910) + * Disable doctor logging on panic (#20847) (#20898) + * Remove calls to load Mirrors in user.Dashboard (#20855) (#20897) + * Update codemirror to 5.65.8 (#20875) + * Rework repo buttons (#20602, #20718) (#20719) +* BUGFIXES + * Ensure delete user deletes all comments (#21067) (#21068) + * Delete unreferenced packages when deleting a package version (#20977) (#21060) + * Redirect if user does not exist on admin pages (#20981) (#21059) + * Set uploadpack.allowFilter etc on gitea serv to enable partial clones with ssh (#20902) (#21058) + * Fix 500 on time in timeline API (#21052) (#21057) + * Fill the specified ref in webhook test payload (#20961) (#21055) + * Add another index for Action table on postgres (#21033) (#21054) + * Fix broken insecureskipverify handling in redis connection uris (#20967) (#21053) + * Add Dev, Peer and Optional dependencies to npm PackageMetadataVersion (#21017) (#21044) + * Do not add links to Posters or Assignees with ID < 0 (#20577) (#21037) + * Fix modified due date message (#20388) (#21032) + * Fix missed sort bug (#21006) + * Fix input.value attr for RequiredClaimName/Value (#20946) (#21001) + * Change review buttons to icons to make space for text (#20934) (#20978) + * Fix download archiver of a commit (#20962) (#20971) + * Return 404 NotFound if requested attachment does not exist (#20886) (#20941) + * Set no-tags in git fetch on compare (#20893) (#20936) + * Allow multiple metadata files for Maven packages (#20674) (#20916) + * Increase Content field size of gpg_key and public_key to MEDIUMTEXT (#20896) (#20911) + * Fix mirror address setting not working (#20850) (#20904) + * Fix push mirror address backend get error Address cause setting page display error (#20593) (#20901) + * Fix panic when an invalid oauth2 name is passed (#20820) (#20900) + * In PushMirrorsIterate and MirrorsIterate if limit is negative do not set it (#20837) (#20899) + * Ensure that graceful start-up is informed of unused SSH listener (#20877) (#20888) + * Pad GPG Key ID with preceding zeroes (#20878) (#20885) + * Fix SQL Query for `SearchTeam` (#20844) (#20872) + * Fix the mode of custom dir to 0700 in docker-rootless (#20861) (#20867) + * Fix UI mis-align for PR commit history (#20845) (#20859) + ## [1.17.1](https://github.com/go-gitea/gitea/releases/tag/1.17.1) - 2022-08-17 * SECURITY diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index f11cf9b11f373..a283f91401839 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -112,11 +112,8 @@ func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) e func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error { return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error { - p, err := archiver.RelativePath() - if err != nil { - return err - } - _, err = storage.Copy(dstStorage, p, storage.RepoArchives, p) + p := archiver.RelativePath() + _, err := storage.Copy(dstStorage, p, storage.RepoArchives, p) return err }) } diff --git a/models/action.go b/models/action.go index 7dbdaa8530cee..f4f71e8440969 100644 --- a/models/action.go +++ b/models/action.go @@ -98,7 +98,14 @@ func (a *Action) TableIndices() []*schemas.Index { actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType) actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted") - return []*schemas.Index{actUserIndex, repoIndex} + indices := []*schemas.Index{actUserIndex, repoIndex} + if setting.Database.UsePostgreSQL { + cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType) + cudIndex.AddColumn("created_unix", "user_id", "is_deleted") + indices = append(indices, cudIndex) + } + + return indices } // GetOpType gets the ActionType of this action. @@ -275,7 +282,7 @@ func (a *Action) GetRefLink() string { return a.GetRepoLink() + "/src/branch/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.BranchPrefix)) case strings.HasPrefix(a.RefName, git.TagPrefix): return a.GetRepoLink() + "/src/tag/" + util.PathEscapeSegments(strings.TrimPrefix(a.RefName, git.TagPrefix)) - case len(a.RefName) == 40 && git.SHAPattern.MatchString(a.RefName): + case len(a.RefName) == 40 && git.IsValidSHAPattern(a.RefName): return a.GetRepoLink() + "/src/commit/" + a.RefName default: // FIXME: we will just assume it's a branch - this was the old way - at some point we may want to enforce that there is always a ref here. diff --git a/models/migrations/v218.go b/models/migrations/v218.go index dee8e5517e54a..e08c6c5b0f5ec 100644 --- a/models/migrations/v218.go +++ b/models/migrations/v218.go @@ -5,6 +5,7 @@ package migrations import ( + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" @@ -37,8 +38,14 @@ func (*improveActionTableIndicesAction) TableIndices() []*schemas.Index { actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType) actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted") - - return []*schemas.Index{actUserIndex, repoIndex} + indices := []*schemas.Index{actUserIndex, repoIndex} + if setting.Database.UsePostgreSQL { + cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType) + cudIndex.AddColumn("created_unix", "user_id", "is_deleted") + indices = append(indices, cudIndex) + } + + return indices } func improveActionTableIndices(x *xorm.Engine) error { diff --git a/models/packages/package.go b/models/packages/package.go index 97cfbc6cad20e..65c852a292e7f 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -214,9 +214,16 @@ func FindUnreferencedPackages(ctx context.Context) ([]*Package, error) { Find(&ps) } -// HasOwnerPackages tests if a user/org has packages +// HasOwnerPackages tests if a user/org has accessible packages func HasOwnerPackages(ctx context.Context, ownerID int64) (bool, error) { - return db.GetEngine(ctx).Where("owner_id = ?", ownerID).Exist(&Package{}) + return db.GetEngine(ctx). + Table("package_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Where(builder.Eq{ + "package_version.is_internal": false, + "package.owner_id": ownerID, + }). + Exist(&PackageVersion{}) } // HasRepositoryPackages tests if a repository has packages diff --git a/models/packages/package_test.go b/models/packages/package_test.go new file mode 100644 index 0000000000000..0bfe6a398bb4c --- /dev/null +++ b/models/packages/package_test.go @@ -0,0 +1,69 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages_test + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + + _ "code.gitea.io/gitea/models" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m, &unittest.TestOptions{ + GiteaRootPath: filepath.Join("..", ".."), + }) +} + +func TestHasOwnerPackages(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User) + + p, err := packages_model.TryInsertPackage(db.DefaultContext, &packages_model.Package{ + OwnerID: owner.ID, + LowerName: "package", + }) + assert.NotNil(t, p) + assert.NoError(t, err) + + // A package without package versions gets automatically cleaned up and should return false + has, err := packages_model.HasOwnerPackages(db.DefaultContext, owner.ID) + assert.False(t, has) + assert.NoError(t, err) + + pv, err := packages_model.GetOrInsertVersion(db.DefaultContext, &packages_model.PackageVersion{ + PackageID: p.ID, + LowerVersion: "internal", + IsInternal: true, + }) + assert.NotNil(t, pv) + assert.NoError(t, err) + + // A package with an internal package version gets automaticaly cleaned up and should return false + has, err = packages_model.HasOwnerPackages(db.DefaultContext, owner.ID) + assert.False(t, has) + assert.NoError(t, err) + + pv, err = packages_model.GetOrInsertVersion(db.DefaultContext, &packages_model.PackageVersion{ + PackageID: p.ID, + LowerVersion: "normal", + IsInternal: false, + }) + assert.NotNil(t, pv) + assert.NoError(t, err) + + // A package with a normal package version should return true + has, err = packages_model.HasOwnerPackages(db.DefaultContext, owner.ID) + assert.True(t, has) + assert.NoError(t, err) +} diff --git a/models/repo.go b/models/repo.go index e9d83f5f32720..d2f3a45940f94 100644 --- a/models/repo.go +++ b/models/repo.go @@ -385,8 +385,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { archivePaths := make([]string, 0, len(archives)) for _, v := range archives { - p, _ := v.RelativePath() - archivePaths = append(archivePaths, p) + archivePaths = append(archivePaths, v.RelativePath()) } if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil { diff --git a/models/repo/archiver.go b/models/repo/archiver.go index dc64cce49ba9d..6a68650fe896b 100644 --- a/models/repo/archiver.go +++ b/models/repo/archiver.go @@ -39,9 +39,9 @@ func init() { db.RegisterModel(new(RepoArchiver)) } -// RelativePath returns relative path -func (archiver *RepoArchiver) RelativePath() (string, error) { - return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()), nil +// RelativePath returns the archive path relative to the archive storage root. +func (archiver *RepoArchiver) RelativePath() string { + return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()) } var delRepoArchiver = new(RepoArchiver) diff --git a/models/user.go b/models/user.go index 49374014aa7db..fb9246070d5aa 100644 --- a/models/user.go +++ b/models/user.go @@ -100,9 +100,9 @@ func DeleteUser(ctx context.Context, u *user_model.User) (err error) { // Delete Comments const batchSize = 50 - for start := 0; ; start += batchSize { + for { comments := make([]*issues_model.Comment, 0, batchSize) - if err = e.Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, start).Find(&comments); err != nil { + if err = e.Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, 0).Find(&comments); err != nil { return err } if len(comments) == 0 { @@ -200,7 +200,7 @@ func DeleteUser(ctx context.Context, u *user_model.User) (err error) { // ***** END: ExternalLoginUser ***** if _, err = e.ID(u.ID).Delete(new(user_model.User)); err != nil { - return fmt.Errorf("Delete: %v", err) + return fmt.Errorf("delete: %v", err) } return nil diff --git a/modules/convert/issue_comment.go b/modules/convert/issue_comment.go index ccc94b249636f..73ad345fa43f9 100644 --- a/modules/convert/issue_comment.go +++ b/modules/convert/issue_comment.go @@ -101,6 +101,12 @@ func ToTimelineComment(c *issues_model.Comment, doer *user_model.User) *api.Time } if c.Time != nil { + err = c.Time.LoadAttributes() + if err != nil { + log.Error("Time.LoadAttributes: %v", err) + return nil + } + comment.TrackedTime = ToTrackedTime(c.Time) } diff --git a/modules/git/git.go b/modules/git/git.go index b8317396c0150..606a73b23097e 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -287,7 +287,20 @@ func syncGitConfig() (err error) { } } - return nil + // By default partial clones are disabled, enable them from git v2.22 + if !setting.Git.DisablePartialClone && CheckGitVersionAtLeast("2.22") == nil { + if err = configSet("uploadpack.allowfilter", "true"); err != nil { + return err + } + err = configSet("uploadpack.allowAnySHA1InWant", "true") + } else { + if err = configUnsetAll("uploadpack.allowfilter", "true"); err != nil { + return err + } + err = configUnsetAll("uploadpack.allowAnySHA1InWant", "true") + } + + return err } // CheckGitVersionAtLeast check git version is at least the constraint version diff --git a/modules/git/ref.go b/modules/git/ref.go index 9fd071ce58dad..2f459148a22d4 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -4,7 +4,10 @@ package git -import "strings" +import ( + "regexp" + "strings" +) const ( // RemotePrefix is the base directory of the remotes information of git. @@ -15,6 +18,29 @@ const ( pullLen = len(PullPrefix) ) +// refNamePatternInvalid is regular expression with unallowed characters in git reference name +// They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere. +// They cannot have question-mark ?, asterisk *, or open bracket [ anywhere +var refNamePatternInvalid = regexp.MustCompile( + `[\000-\037\177 \\~^:?*[]|` + // No absolutely invalid characters + `(?:^[/.])|` + // Not HasPrefix("/") or "." + `(?:/\.)|` + // no "/." + `(?:\.lock$)|(?:\.lock/)|` + // No ".lock/"" or ".lock" at the end + `(?:\.\.)|` + // no ".." anywhere + `(?://)|` + // no "//" anywhere + `(?:@{)|` + // no "@{" + `(?:[/.]$)|` + // no terminal '/' or '.' + `(?:^@$)`) // Not "@" + +// IsValidRefPattern ensures that the provided string could be a valid reference +func IsValidRefPattern(name string) bool { + return !refNamePatternInvalid.MatchString(name) +} + +func SanitizeRefPattern(name string) string { + return refNamePatternInvalid.ReplaceAllString(name, "_") +} + // Reference represents a Git ref. type Reference struct { Name string diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index e528af0ffb88e..efea307b37a2e 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -138,7 +138,7 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id SHA1) (*Co // ConvertToSHA1 returns a Hash object from a potential ID string func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { - if len(commitID) == 40 && SHAPattern.MatchString(commitID) { + if len(commitID) == 40 && IsValidSHAPattern(commitID) { sha1, err := NewIDFromString(commitID) if err == nil { return sha1, nil diff --git a/modules/git/sha1.go b/modules/git/sha1.go index 2da74733df725..15f282c6e4af6 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -19,7 +19,12 @@ const EmptySHA = "0000000000000000000000000000000000000000" const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" // SHAPattern can be used to determine if a string is an valid sha -var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) +var shaPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) + +// IsValidSHAPattern will check if the provided string matches the SHA Pattern +func IsValidSHAPattern(sha string) bool { + return shaPattern.MatchString(sha) +} // MustID always creates a new SHA1 from a [20]byte array with no validation of input. func MustID(b []byte) SHA1 { diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index eaa0dd45e2c4b..dd520cd63f971 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -26,7 +26,7 @@ type PullRequest struct { Updated time.Time Closed *time.Time Labels []*Label - PatchURL string `yaml:"patch_url"` + PatchURL string `yaml:"patch_url"` // SECURITY: This must be safe to download directly from Merged bool MergedTime *time.Time `yaml:"merged_time"` MergeCommitSHA string `yaml:"merge_commit_sha"` @@ -37,6 +37,7 @@ type PullRequest struct { Reactions []*Reaction ForeignIndex int64 Context DownloaderContext `yaml:"-"` + EnsuredSafe bool `yaml:"ensured_safe"` } func (p *PullRequest) GetLocalIndex() int64 { return p.Number } @@ -55,9 +56,9 @@ func (p PullRequest) GetGitRefName() string { // PullRequestBranch represents a pull request branch type PullRequestBranch struct { - CloneURL string `yaml:"clone_url"` - Ref string - SHA string + CloneURL string `yaml:"clone_url"` // SECURITY: This must be safe to download from + Ref string // SECURITY: this must be a git.IsValidRefPattern + SHA string // SECURITY: this must be a git.IsValidSHAPattern RepoName string `yaml:"repo_name"` OwnerName string `yaml:"owner_name"` } diff --git a/modules/migration/release.go b/modules/migration/release.go index cbdf01a3ed1d6..923b3817b038a 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -18,15 +18,16 @@ type ReleaseAsset struct { DownloadCount *int `yaml:"download_count"` Created time.Time Updated time.Time - DownloadURL *string `yaml:"download_url"` + + DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe // if DownloadURL is nil, the function should be invoked - DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe } // Release represents a release type Release struct { - TagName string `yaml:"tag_name"` - TargetCommitish string `yaml:"target_commitish"` + TagName string `yaml:"tag_name"` // SECURITY: This must pass git.IsValidRefPattern + TargetCommitish string `yaml:"target_commitish"` // SECURITY: This must pass git.IsValidRefPattern Name string Body string Draft bool diff --git a/modules/migration/repo.go b/modules/migration/repo.go index d0d62de8da9c4..75622595d9691 100644 --- a/modules/migration/repo.go +++ b/modules/migration/repo.go @@ -12,7 +12,7 @@ type Repository struct { IsPrivate bool `yaml:"is_private"` IsMirror bool `yaml:"is_mirror"` Description string - CloneURL string `yaml:"clone_url"` + CloneURL string `yaml:"clone_url"` // SECURITY: This must be checked to ensure that is safe to be used OriginalURL string `yaml:"original_url"` DefaultBranch string } diff --git a/modules/nosql/manager_redis.go b/modules/nosql/manager_redis.go index b82f899db042f..5e52eb870e895 100644 --- a/modules/nosql/manager_redis.go +++ b/modules/nosql/manager_redis.go @@ -245,7 +245,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config { if len(skipverify) > 0 { skipverify, err := strconv.ParseBool(skipverify) - if err != nil { + if err == nil { tlsConfig.InsecureSkipVerify = skipverify } } @@ -254,7 +254,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config { if len(insecureskipverify) > 0 { insecureskipverify, err := strconv.ParseBool(insecureskipverify) - if err != nil { + if err == nil { tlsConfig.InsecureSkipVerify = insecureskipverify } } diff --git a/modules/nosql/manager_redis_test.go b/modules/nosql/manager_redis_test.go index 3d94532135162..99a8856f1e836 100644 --- a/modules/nosql/manager_redis_test.go +++ b/modules/nosql/manager_redis_test.go @@ -27,6 +27,24 @@ func TestRedisPasswordOpt(t *testing.T) { } } +func TestSkipVerifyOpt(t *testing.T) { + uri, _ := url.Parse("rediss://myredis/0?skipverify=true") + tlsConfig := getRedisTLSOptions(uri) + + if !tlsConfig.InsecureSkipVerify { + t.Fail() + } +} + +func TestInsecureSkipVerifyOpt(t *testing.T) { + uri, _ := url.Parse("rediss://myredis/0?insecureskipverify=true") + tlsConfig := getRedisTLSOptions(uri) + + if !tlsConfig.InsecureSkipVerify { + t.Fail() + } +} + func TestRedisSentinelUsernameOpt(t *testing.T) { uri, _ := url.Parse("redis+sentinel://redis:password@myredis/0?sentinelusername=suser&sentinelpassword=spass") opts := getRedisOptions(uri).Failover() diff --git a/modules/timeutil/timestamp.go b/modules/timeutil/timestamp.go index 1fe8d4fcb187d..9c421914cb584 100644 --- a/modules/timeutil/timestamp.go +++ b/modules/timeutil/timestamp.go @@ -54,6 +54,11 @@ func (ts TimeStamp) AsTime() (tm time.Time) { return ts.AsTimeInLocation(setting.DefaultUILocation) } +// AsLocalTime convert timestamp as time.Time in local location +func (ts TimeStamp) AsLocalTime() time.Time { + return time.Unix(int64(ts), 0) +} + // AsTimeInLocation convert timestamp as time.Time in Local locale func (ts TimeStamp) AsTimeInLocation(loc *time.Location) (tm time.Time) { tm = time.Unix(int64(ts), 0).In(loc) diff --git a/modules/validation/binding.go b/modules/validation/binding.go index 7baa36ccc51ad..f08f632426649 100644 --- a/modules/validation/binding.go +++ b/modules/validation/binding.go @@ -9,6 +9,8 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/git" + "gitea.com/go-chi/binding" "github.com/gobwas/glob" ) @@ -24,30 +26,6 @@ const ( ErrRegexPattern = "RegexPattern" ) -// GitRefNamePatternInvalid is regular expression with unallowed characters in git reference name -// They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere. -// They cannot have question-mark ?, asterisk *, or open bracket [ anywhere -var GitRefNamePatternInvalid = regexp.MustCompile(`[\000-\037\177 \\~^:?*[]+`) - -// CheckGitRefAdditionalRulesValid check name is valid on additional rules -func CheckGitRefAdditionalRulesValid(name string) bool { - // Additional rules as described at https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html - if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || - strings.HasSuffix(name, ".") || strings.Contains(name, "..") || - strings.Contains(name, "//") || strings.Contains(name, "@{") || - name == "@" { - return false - } - parts := strings.Split(name, "/") - for _, part := range parts { - if strings.HasSuffix(part, ".lock") || strings.HasPrefix(part, ".") { - return false - } - } - - return true -} - // AddBindingRules adds additional binding rules func AddBindingRules() { addGitRefNameBindingRule() @@ -67,16 +45,10 @@ func addGitRefNameBindingRule() { IsValid: func(errs binding.Errors, name string, val interface{}) (bool, binding.Errors) { str := fmt.Sprintf("%v", val) - if GitRefNamePatternInvalid.MatchString(str) { + if !git.IsValidRefPattern(str) { errs.Add([]string{name}, ErrGitRefName, "GitRefName") return false, errs } - - if !CheckGitRefAdditionalRulesValid(str) { - errs.Add([]string{name}, ErrGitRefName, "GitRefName") - return false, errs - } - return true, errs }, }) diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index 4b6b803971b7a..763c595152ec1 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -55,15 +55,18 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package metadata := pd.Metadata.(*npm_module.Metadata) return &npm_module.PackageMetadataVersion{ - ID: fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version), - Name: pd.Package.Name, - Version: pd.Version.Version, - Description: metadata.Description, - Author: npm_module.User{Name: metadata.Author}, - Homepage: metadata.ProjectURL, - License: metadata.License, - Dependencies: metadata.Dependencies, - Readme: metadata.Readme, + ID: fmt.Sprintf("%s@%s", pd.Package.Name, pd.Version.Version), + Name: pd.Package.Name, + Version: pd.Version.Version, + Description: metadata.Description, + Author: npm_module.User{Name: metadata.Author}, + Homepage: metadata.ProjectURL, + License: metadata.License, + Dependencies: metadata.Dependencies, + DevDependencies: metadata.DevelopmentDependencies, + PeerDependencies: metadata.PeerDependencies, + OptionalDependencies: metadata.OptionalDependencies, + Readme: metadata.Readme, Dist: npm_module.PackageDistribution{ Shasum: pd.Files[0].Blob.HashSHA1, Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), diff --git a/routers/api/v1/repo/commits.go b/routers/api/v1/repo/commits.go index b196ce97740de..12c329c2030bb 100644 --- a/routers/api/v1/repo/commits.go +++ b/routers/api/v1/repo/commits.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/routers/api/v1/utils" ) @@ -53,7 +52,7 @@ func GetSingleCommit(ctx *context.APIContext) { // "$ref": "#/responses/notFound" sha := ctx.Params(":sha") - if (validation.GitRefNamePatternInvalid.MatchString(sha) || !validation.CheckGitRefAdditionalRulesValid(sha)) && !git.SHAPattern.MatchString(sha) { + if !git.IsValidRefPattern(sha) { ctx.Error(http.StatusUnprocessableEntity, "no valid ref or sha", fmt.Sprintf("no valid ref or sha: %s", sha)) return } diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 2190094bac5c7..57c783d3eedd7 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -8,6 +8,7 @@ package repo import ( "bytes" "encoding/base64" + "errors" "fmt" "io" "net/http" @@ -29,7 +30,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" - "code.gitea.io/gitea/routers/web/repo" + archiver_service "code.gitea.io/gitea/services/repository/archiver" files_service "code.gitea.io/gitea/services/repository/files" ) @@ -294,7 +295,53 @@ func GetArchive(ctx *context.APIContext) { defer gitRepo.Close() } - repo.Download(ctx.Context) + archiveDownload(ctx) +} + +func archiveDownload(ctx *context.APIContext) { + uri := ctx.Params("*") + aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + if err != nil { + if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { + ctx.Error(http.StatusBadRequest, "unknown archive format", err) + } else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) { + ctx.Error(http.StatusNotFound, "unrecognized reference", err) + } else { + ctx.ServerError("archiver_service.NewRequest", err) + } + return + } + + archiver, err := aReq.Await(ctx) + if err != nil { + ctx.ServerError("archiver.Await", err) + return + } + + download(ctx, aReq.GetArchiveName(), archiver) +} + +func download(ctx *context.APIContext, archiveName string, archiver *repo_model.RepoArchiver) { + downloadName := ctx.Repo.Repository.Name + "-" + archiveName + + rPath := archiver.RelativePath() + if setting.RepoArchive.ServeDirect { + // If we have a signed url (S3, object storage), redirect to this directly. + u, err := storage.RepoArchives.URL(rPath, downloadName) + if u != nil && err == nil { + ctx.Redirect(u.String()) + return + } + } + + // If we have matched and access to release or issue + fr, err := storage.RepoArchives.Open(rPath) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) } // GetEditorconfig get editor config of a repository diff --git a/routers/api/v1/repo/hook.go b/routers/api/v1/repo/hook.go index 8a546e581ad5c..b17142c0f647f 100644 --- a/routers/api/v1/repo/hook.go +++ b/routers/api/v1/repo/hook.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/v1/utils" @@ -140,7 +141,7 @@ func TestHook(ctx *context.APIContext) { // required: true // - name: ref // in: query - // description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)" + // description: "The name of the commit/branch/tag, indicates which commit will be loaded to the webhook payload." // type: string // required: false // responses: @@ -153,6 +154,11 @@ func TestHook(ctx *context.APIContext) { return } + ref := git.BranchPrefix + ctx.Repo.Repository.DefaultBranch + if r := ctx.FormTrim("ref"); r != "" { + ref = r + } + hookID := ctx.ParamsInt64(":id") hook, err := utils.GetRepoHook(ctx, ctx.Repo.Repository.ID, hookID) if err != nil { @@ -161,10 +167,12 @@ func TestHook(ctx *context.APIContext) { commit := convert.ToPayloadCommit(ctx.Repo.Repository, ctx.Repo.Commit) + commitID := ctx.Repo.Commit.ID.String() if err := webhook_service.PrepareWebhook(hook, ctx.Repo.Repository, webhook.HookEventPush, &api.PushPayload{ - Ref: git.BranchPrefix + ctx.Repo.Repository.DefaultBranch, - Before: ctx.Repo.Commit.ID.String(), - After: ctx.Repo.Commit.ID.String(), + Ref: ref, + Before: commitID, + After: commitID, + CompareURL: setting.AppURL + ctx.Repo.Repository.ComposeCompareURL(commitID, commitID), Commits: []*api.PayloadCommit{commit}, HeadCommit: commit, Repo: convert.ToRepo(ctx.Repo.Repository, perm.AccessModeNone), diff --git a/routers/api/v1/repo/notes.go b/routers/api/v1/repo/notes.go index bd8e27e40bf40..67f097a424aca 100644 --- a/routers/api/v1/repo/notes.go +++ b/routers/api/v1/repo/notes.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/validation" ) // GetNote Get a note corresponding to a single commit from a repository @@ -47,7 +46,7 @@ func GetNote(ctx *context.APIContext) { // "$ref": "#/responses/notFound" sha := ctx.Params(":sha") - if (validation.GitRefNamePatternInvalid.MatchString(sha) || !validation.CheckGitRefAdditionalRulesValid(sha)) && !git.SHAPattern.MatchString(sha) { + if !git.IsValidRefPattern(sha) { ctx.Error(http.StatusUnprocessableEntity, "no valid ref or sha", fmt.Sprintf("no valid ref or sha: %s", sha)) return } diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index c37ecfd71ea90..39b212bbc316b 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -209,7 +209,11 @@ func NewUserPost(ctx *context.Context) { func prepareUserInfo(ctx *context.Context) *user_model.User { u, err := user_model.GetUserByID(ctx.ParamsInt64(":userid")) if err != nil { - ctx.ServerError("GetUserByID", err) + if user_model.IsErrUserNotExist(err) { + ctx.Redirect(setting.AppSubURL + "/admin/users") + } else { + ctx.ServerError("GetUserByID", err) + } return nil } ctx.Data["User"] = u diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index c2c79e4a0df1c..873884356b579 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -10,7 +10,6 @@ import ( "fmt" "net/http" "strings" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" @@ -21,7 +20,6 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" - "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -389,68 +387,27 @@ func Download(ctx *context.Context) { if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, err.Error()) + } else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) { + ctx.Error(http.StatusNotFound, err.Error()) } else { ctx.ServerError("archiver_service.NewRequest", err) } return } - if aReq == nil { - ctx.Error(http.StatusNotFound) - return - } - archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + archiver, err := aReq.Await(ctx) if err != nil { - ctx.ServerError("models.GetRepoArchiver", err) + ctx.ServerError("archiver.Await", err) return } - if archiver != nil && archiver.Status == repo_model.ArchiverReady { - download(ctx, aReq.GetArchiveName(), archiver) - return - } - - if err := archiver_service.StartArchive(aReq); err != nil { - ctx.ServerError("archiver_service.StartArchive", err) - return - } - - var times int - t := time.NewTicker(time.Second * 1) - defer t.Stop() - for { - select { - case <-graceful.GetManager().HammerContext().Done(): - log.Warn("exit archive download because system stop") - return - case <-t.C: - if times > 20 { - ctx.ServerError("wait download timeout", nil) - return - } - times++ - archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) - if err != nil { - ctx.ServerError("archiver_service.StartArchive", err) - return - } - if archiver != nil && archiver.Status == repo_model.ArchiverReady { - download(ctx, aReq.GetArchiveName(), archiver) - return - } - } - } + download(ctx, aReq.GetArchiveName(), archiver) } func download(ctx *context.Context, archiveName string, archiver *repo_model.RepoArchiver) { downloadName := ctx.Repo.Repository.Name + "-" + archiveName - rPath, err := archiver.RelativePath() - if err != nil { - ctx.ServerError("archiver.RelativePath", err) - return - } - + rPath := archiver.RelativePath() if setting.RepoArchive.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.RepoArchives.URL(rPath, downloadName) diff --git a/routers/web/repo/webhook.go b/routers/web/repo/webhook.go index a9b14ee21f453..2b9f78ab018e0 100644 --- a/routers/web/repo/webhook.go +++ b/routers/web/repo/webhook.go @@ -1271,10 +1271,12 @@ func TestWebhook(ctx *context.Context) { }, } + commitID := commit.ID.String() p := &api.PushPayload{ Ref: git.BranchPrefix + ctx.Repo.Repository.DefaultBranch, - Before: commit.ID.String(), - After: commit.ID.String(), + Before: commitID, + After: commitID, + CompareURL: setting.AppURL + ctx.Repo.Repository.ComposeCompareURL(commitID, commitID), Commits: []*api.PayloadCommit{apiCommit}, HeadCommit: apiCommit, Repo: convert.ToRepo(ctx.Repo.Repository, perm.AccessModeNone), diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index bb74c0a49d1dd..edeb276773452 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -107,9 +107,24 @@ func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, re commitMap: make(map[string]string), } + log.Trace("Create Codebase downloader. BaseURL: %s Project: %s RepoName: %s", baseURL, project, repoName) return downloader } +// String implements Stringer +func (d *CodebaseDownloader) String() string { + return fmt.Sprintf("migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) +} + +// ColorFormat provides a basic color format for a GogsDownloader +func (d *CodebaseDownloader) ColorFormat(s fmt.State) { + if d == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) +} + // FormatCloneURL add authentication into remote URLs func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) { return opts.CloneAddr, nil @@ -451,8 +466,8 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` - SourceRef string `xml:"source-ref"` - TargetRef string `xml:"target-ref"` + SourceRef string `xml:"source-ref"` // NOTE: from the documentation these are actually just branches NOT full refs + TargetRef string `xml:"target-ref"` // NOTE: from the documentation these are actually just branches NOT full refs Subject string `xml:"subject"` Status string `xml:"status"` UserID struct { @@ -564,6 +579,9 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq Comments: comments[1:], }, }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) } return pullRequests, true, nil diff --git a/services/migrations/common.go b/services/migrations/common.go new file mode 100644 index 0000000000000..305ae89b2de54 --- /dev/null +++ b/services/migrations/common.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "strings" + + admin_model "code.gitea.io/gitea/models/admin" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" +) + +// WarnAndNotice will log the provided message and send a repository notice +func WarnAndNotice(fmtStr string, args ...interface{}) { + log.Warn(fmtStr, args...) + if err := admin_model.CreateRepositoryNotice(fmt.Sprintf(fmtStr, args...)); err != nil { + log.Error("create repository notice failed: ", err) + } +} + +func hasBaseURL(toCheck, baseURL string) bool { + if len(baseURL) > 0 && baseURL[len(baseURL)-1] != '/' { + baseURL += "/" + } + return strings.HasPrefix(toCheck, baseURL) +} + +// CheckAndEnsureSafePR will check that a given PR is safe to download +func CheckAndEnsureSafePR(pr *base.PullRequest, commonCloneBaseURL string, g base.Downloader) bool { + valid := true + // SECURITY: the patchURL must be checked to have the same baseURL as the current to prevent open redirect + if pr.PatchURL != "" && !hasBaseURL(pr.PatchURL, commonCloneBaseURL) { + // TODO: Should we check that this url has the expected format for a patch url? + WarnAndNotice("PR #%d in %s has invalid PatchURL: %s baseURL: %s", pr.Number, g, pr.PatchURL, commonCloneBaseURL) + pr.PatchURL = "" + valid = false + } + + // SECURITY: the headCloneURL must be checked to have the same baseURL as the current to prevent open redirect + if pr.Head.CloneURL != "" && !hasBaseURL(pr.Head.CloneURL, commonCloneBaseURL) { + // TODO: Should we check that this url has the expected format for a patch url? + WarnAndNotice("PR #%d in %s has invalid HeadCloneURL: %s baseURL: %s", pr.Number, g, pr.Head.CloneURL, commonCloneBaseURL) + pr.Head.CloneURL = "" + valid = false + } + + // SECURITY: SHAs Must be a SHA + if pr.MergeCommitSHA != "" && !git.IsValidSHAPattern(pr.MergeCommitSHA) { + WarnAndNotice("PR #%d in %s has invalid MergeCommitSHA: %s", pr.Number, g, pr.MergeCommitSHA) + pr.MergeCommitSHA = "" + } + if pr.Head.SHA != "" && !git.IsValidSHAPattern(pr.Head.SHA) { + WarnAndNotice("PR #%d in %s has invalid HeadSHA: %s", pr.Number, g, pr.Head.SHA) + pr.Head.SHA = "" + valid = false + } + if pr.Base.SHA != "" && !git.IsValidSHAPattern(pr.Base.SHA) { + WarnAndNotice("PR #%d in %s has invalid BaseSHA: %s", pr.Number, g, pr.Base.SHA) + pr.Base.SHA = "" + valid = false + } + + // SECURITY: Refs must be valid refs or SHAs + if pr.Head.Ref != "" && !git.IsValidRefPattern(pr.Head.Ref) { + WarnAndNotice("PR #%d in %s has invalid HeadRef: %s", pr.Number, g, pr.Head.Ref) + pr.Head.Ref = "" + valid = false + } + if pr.Base.Ref != "" && !git.IsValidRefPattern(pr.Base.Ref) { + WarnAndNotice("PR #%d in %s has invalid BaseRef: %s", pr.Number, g, pr.Base.Ref) + pr.Base.Ref = "" + valid = false + } + + pr.EnsuredSafe = true + + return valid +} diff --git a/services/migrations/dump.go b/services/migrations/dump.go index f040c6c6630fe..2dc35b9d9fbef 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -12,7 +12,6 @@ import ( "net/http" "net/url" "os" - "path" "path/filepath" "strconv" "strings" @@ -26,6 +25,8 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" + "github.com/google/uuid" + "gopkg.in/yaml.v2" ) @@ -47,7 +48,7 @@ type RepositoryDumper struct { reviewFiles map[int64]*os.File gitRepo *git.Repository - prHeadCache map[string]struct{} + prHeadCache map[string]string } // NewRepositoryDumper creates an gitea Uploader @@ -62,7 +63,7 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin baseDir: baseDir, repoOwner: repoOwner, repoName: repoName, - prHeadCache: make(map[string]struct{}), + prHeadCache: make(map[string]string), commentFiles: make(map[int64]*os.File), reviewFiles: make(map[int64]*os.File), }, nil @@ -296,8 +297,10 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { } for _, asset := range release.Assets { attachLocalPath := filepath.Join(attachDir, asset.Name) - // download attachment + // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here + // ... we must assume that they are safe and simply download the attachment + // download attachment err := func(attachPath string) error { var rc io.ReadCloser var err error @@ -317,7 +320,7 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { fw, err := os.Create(attachPath) if err != nil { - return fmt.Errorf("Create: %v", err) + return fmt.Errorf("create: %w", err) } defer fw.Close() @@ -385,27 +388,29 @@ func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, } for number, items := range itemsMap { - var err error - itemFile := itemFiles[number] - if itemFile == nil { - itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number))) - if err != nil { - return err - } - itemFiles[number] = itemFile - } - - bs, err := yaml.Marshal(items) - if err != nil { + if err := g.encodeItems(number, items, dir, itemFiles); err != nil { return err } + } + + return nil +} - if _, err := itemFile.Write(bs); err != nil { +func (g *RepositoryDumper) encodeItems(number int64, items []interface{}, dir string, itemFiles map[int64]*os.File) error { + itemFile := itemFiles[number] + if itemFile == nil { + var err error + itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number))) + if err != nil { return err } + itemFiles[number] = itemFile } - return nil + encoder := yaml.NewEncoder(itemFile) + defer encoder.Close() + + return encoder.Encode(items) } // CreateComments creates comments of issues @@ -418,102 +423,175 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { return g.createItems(g.commentDir(), g.commentFiles, commentsMap) } -// CreatePullRequests creates pull requests -func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { - for _, pr := range prs { - // download patch file - err := func() error { - u, err := g.setURLToken(pr.PatchURL) - if err != nil { - return err - } - resp, err := http.Get(u) - if err != nil { - return err - } - defer resp.Body.Close() - pullDir := filepath.Join(g.gitPath(), "pulls") - if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { - return err - } - fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)) - f, err := os.Create(fPath) - if err != nil { - return err - } - defer f.Close() - if _, err = io.Copy(f, resp.Body); err != nil { - return err - } - pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) +func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { + // SECURITY: this pr must have been ensured safe + if !pr.EnsuredSafe { + log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName) + return fmt.Errorf("unsafe PR #%d", pr.Number) + } + // First we download the patch file + err := func() error { + // if the patchURL is empty there is nothing to download + if pr.PatchURL == "" { return nil - }() + } + + // SECURITY: We will assume that the pr.PatchURL has been checked + // pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe + u, err := g.setURLToken(pr.PatchURL) if err != nil { return err } - // set head information - pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) - if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { + // SECURITY: We will assume that the pr.PatchURL has been checked + // pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe + resp, err := http.Get(u) // TODO: This probably needs to use the downloader as there may be rate limiting issues here + if err != nil { return err } - p, err := os.Create(filepath.Join(pullHead, "head")) - if err != nil { + defer resp.Body.Close() + pullDir := filepath.Join(g.gitPath(), "pulls") + if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err } - _, err = p.WriteString(pr.Head.SHA) - p.Close() + fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)) + f, err := os.Create(fPath) if err != nil { return err } + defer f.Close() - if pr.IsForkPullRequest() && pr.State != "closed" { - if pr.Head.OwnerName != "" { - remote := pr.Head.OwnerName - _, ok := g.prHeadCache[remote] - if !ok { - // git remote add - // TODO: how to handle private CloneURL? - err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) - if err != nil { - log.Error("AddRemote failed: %s", err) - } else { - g.prHeadCache[remote] = struct{}{} - ok = true - } - } + // TODO: Should there be limits on the size of this file? + if _, err = io.Copy(f, resp.Body); err != nil { + return err + } + pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) - if ok { - _, _, err = git.NewCommand(g.ctx, "fetch", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.gitPath()}) - if err != nil { - log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) - } else { - // a new branch name with will be created to as new head branch - ref := path.Join(pr.Head.OwnerName, pr.Head.Ref) - headBranch := filepath.Join(g.gitPath(), "refs", "heads", ref) - if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return err - } - b, err := os.Create(headBranch) - if err != nil { - return err - } - _, err = b.WriteString(pr.Head.SHA) - b.Close() - if err != nil { - return err - } - pr.Head.Ref = ref - } + return nil + }() + if err != nil { + log.Error("PR #%d in %s/%s unable to download patch: %v", pr.Number, g.repoOwner, g.repoName, err) + return err + } + + isFork := pr.IsForkPullRequest() + + // Even if it's a forked repo PR, we have to change head info as the same as the base info + oldHeadOwnerName := pr.Head.OwnerName + pr.Head.OwnerName, pr.Head.RepoName = pr.Base.OwnerName, pr.Base.RepoName + + if !isFork || pr.State == "closed" { + return nil + } + + // OK we want to fetch the current head as a branch from its CloneURL + + // 1. Is there a head clone URL available? + // 2. Is there a head ref available? + if pr.Head.CloneURL == "" || pr.Head.Ref == "" { + // Set head information if pr.Head.SHA is available + if pr.Head.SHA != "" { + _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + if err != nil { + log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) + } + } + return nil + } + + // 3. We need to create a remote for this clone url + // ... maybe we already have a name for this remote + remote, ok := g.prHeadCache[pr.Head.CloneURL+":"] + if !ok { + // ... let's try ownername as a reasonable name + remote = oldHeadOwnerName + if !git.IsValidRefPattern(remote) { + // ... let's try something less nice + remote = "head-pr-" + strconv.FormatInt(pr.Number, 10) + } + // ... now add the remote + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if err != nil { + log.Error("PR #%d in %s/%s AddRemote[%s] failed: %v", pr.Number, g.repoOwner, g.repoName, remote, err) + } else { + g.prHeadCache[pr.Head.CloneURL+":"] = remote + ok = true + } + } + if !ok { + // Set head information if pr.Head.SHA is available + if pr.Head.SHA != "" { + _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + if err != nil { + log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) + } + } + + return nil + } + + // 4. Check if we already have this ref? + localRef, ok := g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] + if !ok { + // ... We would normally name this migrated branch as / but we need to ensure that is safe + localRef = git.SanitizeRefPattern(oldHeadOwnerName + "/" + pr.Head.Ref) + + // ... Now we must assert that this does not exist + if g.gitRepo.IsBranchExist(localRef) { + localRef = "head-pr-" + strconv.FormatInt(pr.Number, 10) + "/" + localRef + i := 0 + for g.gitRepo.IsBranchExist(localRef) { + if i > 5 { + // ... We tried, we really tried but this is just a seriously unfriendly repo + return fmt.Errorf("unable to create unique local reference from %s", pr.Head.Ref) } + // OK just try some uuids! + localRef = git.SanitizeRefPattern("head-pr-" + strconv.FormatInt(pr.Number, 10) + uuid.New().String()) + i++ } } - // whatever it's a forked repo PR, we have to change head info as the same as the base info - pr.Head.OwnerName = pr.Base.OwnerName - pr.Head.RepoName = pr.Base.RepoName + + fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef + if strings.HasPrefix(fetchArg, "-") { + fetchArg = git.BranchPrefix + fetchArg + } + + _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + if err != nil { + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + // We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR + // (This last step will likely fail but we should try to do as much as we can.) + } else { + // Cache the localRef as the Head.Ref - if we've failed we can always try again. + g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] = localRef + } } + // Set the pr.Head.Ref to the localRef + pr.Head.Ref = localRef + + // 5. Now if pr.Head.SHA == "" we should recover this to the head of this branch + if pr.Head.SHA == "" { + headSha, err := g.gitRepo.GetBranchCommitID(localRef) + if err != nil { + log.Error("unable to get head SHA of local head for PR #%d from %s in %s/%s. Error: %v", pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) + return nil + } + pr.Head.SHA = headSha + } + if pr.Head.SHA != "" { + _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + if err != nil { + log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) + } + } + + return nil +} + +// CreatePullRequests creates pull requests +func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { var err error if g.pullrequestFile == nil { if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { @@ -525,16 +603,22 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { } } - bs, err := yaml.Marshal(prs) - if err != nil { - return err - } + encoder := yaml.NewEncoder(g.pullrequestFile) + defer encoder.Close() - if _, err := g.pullrequestFile.Write(bs); err != nil { - return err + count := 0 + for i := 0; i < len(prs); i++ { + pr := prs[i] + if err := g.handlePullRequest(pr); err != nil { + log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err) + continue + } + prs[count] = pr + count++ } + prs = prs[:count] - return nil + return encoder.Encode(prs) } // CreateReviews create pull request reviews diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index 92b6cac73806b..21d8c672ddfa2 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -6,9 +6,11 @@ package migrations import ( "context" + "fmt" "net/url" "strings" + "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/structs" ) @@ -37,6 +39,7 @@ func (f *GitBucketDownloaderFactory) New(ctx context.Context, opts base.MigrateO oldOwner := fields[1] oldName := strings.TrimSuffix(fields[2], ".git") + log.Trace("Create GitBucket downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, oldOwner, oldName) return NewGitBucketDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil } @@ -51,6 +54,20 @@ type GitBucketDownloader struct { *GithubDownloaderV3 } +// String implements Stringer +func (g *GitBucketDownloader) String() string { + return fmt.Sprintf("migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + +// ColorFormat provides a basic color format for a GitBucketDownloader +func (g *GitBucketDownloader) ColorFormat(s fmt.State) { + if g == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from gitbucket server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + // NewGitBucketDownloader creates a GitBucket downloader func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GitBucketDownloader { githubDownloader := NewGithubDownloaderV3(ctx, baseURL, userName, password, token, repoOwner, repoName) diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 4ad55894ee28f..49358abebcfb7 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -14,7 +14,6 @@ import ( "strings" "time" - admin_model "code.gitea.io/gitea/models/admin" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/structs" @@ -71,6 +70,7 @@ type GiteaDownloader struct { base.NullDownloader ctx context.Context client *gitea_sdk.Client + baseURL string repoOwner string repoName string pagination bool @@ -116,6 +116,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo return &GiteaDownloader{ ctx: ctx, client: giteaClient, + baseURL: baseURL, repoOwner: path[0], repoName: path[1], pagination: paginationSupport, @@ -128,6 +129,20 @@ func (g *GiteaDownloader) SetContext(ctx context.Context) { g.ctx = ctx } +// String implements Stringer +func (g *GiteaDownloader) String() string { + return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + +// ColorFormat provides a basic color format for a GiteaDownloader +func (g *GiteaDownloader) ColorFormat(s fmt.State) { + if g == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + // GetRepoInfo returns a repository information func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { if g == nil { @@ -283,6 +298,12 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele if err != nil { return nil, err } + + if !hasBaseURL(asset.DownloadURL, g.baseURL) { + WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.ID, g, asset.DownloadURL) + return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil + } + // FIXME: for a private download? req, err := http.NewRequest("GET", asset.DownloadURL, nil) if err != nil { @@ -402,11 +423,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err reactions, err := g.getIssueReactions(issue.Index) if err != nil { - log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err) - if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) - } + WarnAndNotice("Unable to load reactions during migrating issue #%d in %s. Error: %v", issue.Index, g, err) } var assignees []string @@ -464,11 +481,7 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err) - if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", commentable.GetForeignIndex(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) - } + WarnAndNotice("Unable to load comment reactions during migrating issue #%d for comment %d in %s. Error: %v", commentable.GetForeignIndex(), comment.ID, g, err) } allComments = append(allComments, &base.Comment{ @@ -543,11 +556,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques reactions, err := g.getIssueReactions(pr.Index) if err != nil { - log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err) - if err2 := admin_model.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) - } + WarnAndNotice("Unable to load reactions during migrating pull #%d in %s. Error: %v", pr.Index, g, err) } var assignees []string @@ -604,6 +613,8 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques }, ForeignIndex: pr.Index, }) + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) } isEnd := len(prs) < perPage diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index c7a6f9b02f2c3..21f587604952f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "strconv" "strings" @@ -32,7 +33,7 @@ import ( "code.gitea.io/gitea/modules/uri" "code.gitea.io/gitea/services/pull" - gouuid "github.com/google/uuid" + "github.com/google/uuid" ) var _ base.Uploader = &GiteaLocalUploader{} @@ -48,7 +49,7 @@ type GiteaLocalUploader struct { milestones map[string]int64 issues map[int64]*issues_model.Issue gitRepo *git.Repository - prHeadCache map[string]struct{} + prHeadCache map[string]string sameApp bool userMap map[int64]int64 // external user id mapping to user id prCache map[int64]*issues_model.PullRequest @@ -65,7 +66,7 @@ func NewGiteaLocalUploader(ctx context.Context, doer *user_model.User, repoOwner labels: make(map[string]*issues_model.Label), milestones: make(map[string]int64), issues: make(map[int64]*issues_model.Issue), - prHeadCache: make(map[string]struct{}), + prHeadCache: make(map[string]string), userMap: make(map[int64]int64), prCache: make(map[int64]*issues_model.PullRequest), } @@ -125,7 +126,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate Mirror: repo.IsMirror, LFS: opts.LFS, LFSEndpoint: opts.LFSEndpoint, - CloneAddr: repo.CloneURL, + CloneAddr: repo.CloneURL, // SECURITY: we will assume that this has already been checked Private: repo.IsPrivate, Wiki: opts.Wiki, Releases: opts.Releases, // if didn't get releases, then sync them from tags @@ -150,13 +151,15 @@ func (g *GiteaLocalUploader) Close() { // CreateTopics creates topics func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { - // ignore topics to long for the db + // Ignore topics too long for the db c := 0 - for i := range topics { - if len(topics[i]) <= 50 { - topics[c] = topics[i] - c++ + for _, topic := range topics { + if len(topic) > 50 { + continue } + + topics[c] = topic + c++ } topics = topics[:c] return repo_model.SaveTopics(g.repo.ID, topics...) @@ -217,11 +220,17 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { lbs := make([]*issues_model.Label, 0, len(labels)) for _, label := range labels { + // We must validate color here: + if !issues_model.LabelColorPattern.MatchString("#" + label.Color) { + log.Warn("Invalid label color: #%s for label: %s in migration to %s/%s", label.Color, label.Name, g.repoOwner, g.repoName) + label.Color = "ffffff" + } + lbs = append(lbs, &issues_model.Label{ RepoID: g.repo.ID, Name: label.Name, Description: label.Description, - Color: fmt.Sprintf("#%s", label.Color), + Color: "#" + label.Color, }) } @@ -247,6 +256,16 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { } } + // SECURITY: The TagName must be a valid git ref + if release.TagName != "" && !git.IsValidRefPattern(release.TagName) { + release.TagName = "" + } + + // SECURITY: The TargetCommitish must be a valid git ref + if release.TargetCommitish != "" && !git.IsValidRefPattern(release.TargetCommitish) { + release.TargetCommitish = "" + } + rel := models.Release{ RepoID: g.repo.ID, TagName: release.TagName, @@ -288,14 +307,15 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { } } attach := repo_model.Attachment{ - UUID: gouuid.New().String(), + UUID: uuid.New().String(), Name: asset.Name, DownloadCount: int64(*asset.DownloadCount), Size: int64(*asset.Size), CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), } - // download attachment + // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here + // ... we must assume that they are safe and simply download the attachment err := func() error { // asset.DownloadURL maybe a local file var rc io.ReadCloser @@ -365,6 +385,12 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } } + // SECURITY: issue.Ref needs to be a valid reference + if !git.IsValidRefPattern(issue.Ref) { + log.Warn("Invalid issue.Ref[%s] in issue #%d in %s/%s", issue.Ref, issue.Number, g.repoOwner, g.repoName) + issue.Ref = "" + } + is := issues_model.Issue{ RepoID: g.repo.ID, Repo: g.repo, @@ -496,102 +522,151 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error } func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { - // download patch file + // SECURITY: this pr must have been must have been ensured safe + if !pr.EnsuredSafe { + log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName) + return "", fmt.Errorf("the PR[%d] was not checked for safety", pr.Number) + } + + // Anonymous function to download the patch file (allows us to use defer) err = func() error { + // if the patchURL is empty there is nothing to download if pr.PatchURL == "" { return nil } - // pr.PatchURL maybe a local file - ret, err := uri.Open(pr.PatchURL) + + // SECURITY: We will assume that the pr.PatchURL has been checked + // pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe + ret, err := uri.Open(pr.PatchURL) // TODO: This probably needs to use the downloader as there may be rate limiting issues here if err != nil { return err } defer ret.Close() + pullDir := filepath.Join(g.repo.RepoPath(), "pulls") if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { return err } + f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) if err != nil { return err } defer f.Close() + + // TODO: Should there be limits on the size of this file? _, err = io.Copy(f, ret) + return err }() if err != nil { return "", err } - // set head information - pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) - if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { - return "", err - } - p, err := os.Create(filepath.Join(pullHead, "head")) - if err != nil { - return "", err - } - _, err = p.WriteString(pr.Head.SHA) - p.Close() - if err != nil { - return "", err - } - head = "unknown repository" if pr.IsForkPullRequest() && pr.State != "closed" { - if pr.Head.OwnerName != "" { - remote := pr.Head.OwnerName - _, ok := g.prHeadCache[remote] - if !ok { - // git remote add - err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) - if err != nil { - log.Error("AddRemote failed: %s", err) - } else { - g.prHeadCache[remote] = struct{}{} - ok = true - } + // OK we want to fetch the current head as a branch from its CloneURL + + // 1. Is there a head clone URL available? + // 2. Is there a head ref available? + if pr.Head.CloneURL == "" || pr.Head.Ref == "" { + return head, nil + } + + // 3. We need to create a remote for this clone url + // ... maybe we already have a name for this remote + remote, ok := g.prHeadCache[pr.Head.CloneURL+":"] + if !ok { + // ... let's try ownername as a reasonable name + remote = pr.Head.OwnerName + if !git.IsValidRefPattern(remote) { + // ... let's try something less nice + remote = "head-pr-" + strconv.FormatInt(pr.Number, 10) + } + // ... now add the remote + err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) + if err != nil { + log.Error("PR #%d in %s/%s AddRemote[%s] failed: %v", pr.Number, g.repoOwner, g.repoName, remote, err) + } else { + g.prHeadCache[pr.Head.CloneURL+":"] = remote + ok = true } + } + if !ok { + return head, nil + } - if ok { - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, pr.Head.Ref).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) - if err != nil { - log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) - } else { - headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) - if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return "", err - } - b, err := os.Create(headBranch) - if err != nil { - return "", err - } - _, err = b.WriteString(pr.Head.SHA) - b.Close() - if err != nil { - return "", err + // 4. Check if we already have this ref? + localRef, ok := g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] + if !ok { + // ... We would normally name this migrated branch as / but we need to ensure that is safe + localRef = git.SanitizeRefPattern(pr.Head.OwnerName + "/" + pr.Head.Ref) + + // ... Now we must assert that this does not exist + if g.gitRepo.IsBranchExist(localRef) { + localRef = "head-pr-" + strconv.FormatInt(pr.Number, 10) + "/" + localRef + i := 0 + for g.gitRepo.IsBranchExist(localRef) { + if i > 5 { + // ... We tried, we really tried but this is just a seriously unfriendly repo + return head, nil } - head = pr.Head.OwnerName + "/" + pr.Head.Ref + // OK just try some uuids! + localRef = git.SanitizeRefPattern("head-pr-" + strconv.FormatInt(pr.Number, 10) + uuid.New().String()) + i++ } } + + fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef + if strings.HasPrefix(fetchArg, "-") { + fetchArg = git.BranchPrefix + fetchArg + } + + _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags", "--", remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + if err != nil { + log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) + return head, nil + } + g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] = localRef + head = localRef } - } else { + + // 5. Now if pr.Head.SHA == "" we should recover this to the head of this branch + if pr.Head.SHA == "" { + headSha, err := g.gitRepo.GetBranchCommitID(localRef) + if err != nil { + log.Error("unable to get head SHA of local head for PR #%d from %s in %s/%s. Error: %v", pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) + return head, nil + } + pr.Head.SHA = headSha + } + + _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + if err != nil { + return "", err + } + + return head, nil + } + + if pr.Head.Ref != "" { head = pr.Head.Ref - // Ensure the closed PR SHA still points to an existing ref + } + + // Ensure the closed PR SHA still points to an existing ref + if pr.Head.SHA == "" { + // The SHA is empty + log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName) + } else { _, _, err = git.NewCommand(g.ctx, "rev-list", "--quiet", "-1", pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { - if pr.Head.SHA != "" { - // Git update-ref remove bad references with a relative path - log.Warn("Deprecated local head, removing : %v", pr.Head.SHA) - err = g.gitRepo.RemoveReference(pr.GetGitRefName()) - } else { - // The SHA is empty, remove the head file - log.Warn("Empty reference, removing : %v", pullHead) - err = os.Remove(filepath.Join(pullHead, "head")) - } + // Git update-ref remove bad references with a relative path + log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName()) + } else { + // set head information + _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref", pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { - log.Error("Cannot remove local head ref, %v", err) + log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) } } } @@ -615,6 +690,20 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model return nil, fmt.Errorf("updateGitForPullRequest: %w", err) } + // Now we may need to fix the mergebase + if pr.Base.SHA == "" { + if pr.Base.Ref != "" && pr.Head.SHA != "" { + // A PR against a tag base does not make sense - therefore pr.Base.Ref must be a branch + // TODO: should we be checking for the refs/heads/ prefix on the pr.Base.Ref? (i.e. are these actually branches or refs) + pr.Base.SHA, _, err = g.gitRepo.GetMergeBase("", git.BranchPrefix+pr.Base.Ref, pr.Head.SHA) + if err != nil { + log.Error("Cannot determine the merge base for PR #%d in %s/%s. Error: %v", pr.Number, g.repoOwner, g.repoName, err) + } + } else { + log.Error("Cannot determine the merge base for PR #%d in %s/%s. Not enough information", pr.Number, g.repoOwner, g.repoName) + } + } + if pr.Created.IsZero() { if pr.Closed != nil { pr.Created = *pr.Closed @@ -728,6 +817,8 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { return err } + cms = append(cms, &cm) + // get pr pr, ok := g.prCache[issue.ID] if !ok { @@ -738,6 +829,17 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } g.prCache[issue.ID] = pr } + if pr.MergeBase == "" { + // No mergebase -> no basis for any patches + log.Warn("PR #%d in %s/%s: does not have a merge base, all review comments will be ignored", pr.Index, g.repoOwner, g.repoName) + continue + } + + headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName()) + if err != nil { + log.Warn("PR #%d GetRefCommitID[%s] in %s/%s: %v, all review comments will be ignored", pr.Index, pr.GetGitRefName(), g.repoOwner, g.repoName, err) + continue + } for _, comment := range review.Comments { line := comment.Line @@ -746,11 +848,9 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } else { _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk) } - headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName()) - if err != nil { - log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err) - continue - } + + // SECURITY: The TreePath must be cleaned! + comment.TreePath = path.Clean("/" + comment.TreePath)[1:] var patch string reader, writer := io.Pipe() @@ -775,6 +875,11 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { comment.UpdatedAt = comment.CreatedAt } + if !git.IsValidSHAPattern(comment.CommitID) { + log.Warn("Invalid comment CommitID[%s] on comment[%d] in PR #%d of %s/%s replaced with %s", comment.CommitID, pr.Index, g.repoOwner, g.repoName, headCommitID) + comment.CommitID = headCommitID + } + c := issues_model.Comment{ Type: issues_model.CommentTypeCode, IssueID: issue.ID, @@ -793,8 +898,6 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { cm.Comments = append(cm.Comments, &c) } - - cms = append(cms, &cm) } return issues_model.InsertReviews(cms) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index c30756017c361..18e3b6cb50f0e 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -391,7 +391,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, }, assertContent: func(t *testing.T, content string) { - assert.Contains(t, content, "AddRemote failed") + assert.Contains(t, content, "AddRemote") }, }, { @@ -440,7 +440,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, }, assertContent: func(t *testing.T, content string) { - assert.Contains(t, content, "Empty reference, removing") + assert.Contains(t, content, "Empty reference") assert.NotContains(t, content, "Cannot remove local head") }, }, @@ -468,7 +468,6 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { }, assertContent: func(t *testing.T, content string) { assert.Contains(t, content, "Deprecated local head") - assert.Contains(t, content, "Cannot remove local head") }, }, { @@ -505,6 +504,8 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { logger.SetLogger("buffer", "buffer", "{}") defer logger.DelLogger("buffer") + testCase.pr.EnsuredSafe = true + head, err := uploader.updateGitForPullRequest(&testCase.pr) assert.NoError(t, err) assert.EqualValues(t, testCase.head, head) diff --git a/services/migrations/github.go b/services/migrations/github.go index 5f5b430fa9e07..0ffdbb042af20 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -51,7 +51,7 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp oldOwner := fields[1] oldName := strings.TrimSuffix(fields[2], ".git") - log.Trace("Create github downloader: %s/%s", oldOwner, oldName) + log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName) return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil } @@ -67,6 +67,7 @@ type GithubDownloaderV3 struct { base.NullDownloader ctx context.Context clients []*github.Client + baseURL string repoOwner string repoName string userName string @@ -81,6 +82,7 @@ type GithubDownloaderV3 struct { func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { downloader := GithubDownloaderV3{ userName: userName, + baseURL: baseURL, password: password, ctx: ctx, repoOwner: repoOwner, @@ -118,6 +120,20 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok return &downloader } +// String implements Stringer +func (g *GithubDownloaderV3) String() string { + return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + +// ColorFormat provides a basic color format for a GithubDownloader +func (g *GithubDownloaderV3) ColorFormat(s fmt.State) { + if g == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) { githubClient := github.NewClient(client) if baseURL != "https://github.com" { @@ -322,33 +338,44 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) Updated: asset.UpdatedAt.Time, DownloadFunc: func() (io.ReadCloser, error) { g.waitAndPickClient() - asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) + readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) if err != nil { return nil, err } if err := g.RefreshRate(); err != nil { log.Error("g.getClient().RateLimits: %s", err) } - if asset == nil { - if redirectURL != "" { - g.waitAndPickClient() - req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) - if err != nil { - return nil, err - } - resp, err := httpClient.Do(req) - err1 := g.RefreshRate() - if err1 != nil { - log.Error("g.getClient().RateLimits: %s", err1) - } - if err != nil { - return nil, err - } - return resp.Body, nil - } - return nil, fmt.Errorf("No release asset found for %d", assetID) + + if readCloser != nil { + return readCloser, nil + } + + if redirectURL == "" { + return nil, fmt.Errorf("no release asset found for %d", assetID) + } + + // Prevent open redirect + if !hasBaseURL(redirectURL, g.baseURL) && + !hasBaseURL(redirectURL, "https://objects.githubusercontent.com/") { + WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.GetID(), g, redirectURL) + + return io.NopCloser(strings.NewReader(redirectURL)), nil + } + + g.waitAndPickClient() + req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) + if err != nil { + return nil, err } - return asset, nil + resp, err := httpClient.Do(req) + err1 := g.RefreshRate() + if err1 != nil { + log.Error("g.RefreshRate(): %s", err1) + } + if err != nil { + return nil, err + } + return resp.Body, nil }, }) } @@ -697,7 +724,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq SHA: pr.GetHead().GetSHA(), OwnerName: pr.GetHead().GetUser().GetLogin(), RepoName: pr.GetHead().GetRepo().GetName(), - CloneURL: pr.GetHead().GetRepo().GetCloneURL(), + CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here }, Base: base.PullRequestBranch{ Ref: pr.GetBase().GetRef(), @@ -705,10 +732,13 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq RepoName: pr.GetBase().GetRepo().GetName(), OwnerName: pr.GetBase().GetUser().GetLogin(), }, - PatchURL: pr.GetPatchURL(), + PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here Reactions: reactions, ForeignIndex: int64(*pr.Number), }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) } return allPRs, len(prs) < perPage, nil diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 549e3cb659c9a..e25169b2ff31f 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -63,6 +63,7 @@ type GitlabDownloader struct { base.NullDownloader ctx context.Context client *gitlab.Client + baseURL string repoID int repoName string issueCount int64 @@ -124,12 +125,27 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw return &GitlabDownloader{ ctx: ctx, client: gitlabClient, + baseURL: baseURL, repoID: gr.ID, repoName: gr.Name, maxPerPage: 100, }, nil } +// String implements Stringer +func (g *GitlabDownloader) String() string { + return fmt.Sprintf("migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName) +} + +// ColorFormat provides a basic color format for a GitlabDownloader +func (g *GitlabDownloader) ColorFormat(s fmt.State) { + if g == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from gitlab server %s [%d]/%s", g.baseURL, g.repoID, g.repoName) +} + // SetContext set context func (g *GitlabDownloader) SetContext(ctx context.Context) { g.ctx = ctx @@ -307,6 +323,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea return nil, err } + if !hasBaseURL(link.URL, g.baseURL) { + WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.ID, g, link.URL) + return io.NopCloser(strings.NewReader(link.URL)), nil + } + req, err := http.NewRequest("GET", link.URL, nil) if err != nil { return nil, err @@ -610,6 +631,9 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque ForeignIndex: int64(pr.IID), Context: gitlabIssueContext{IsMergeRequest: true}, }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) } return allPRs, len(prs) < perPage, nil diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index a28033218eac5..46cc3ca416461 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -73,6 +73,20 @@ type GogsDownloader struct { transport http.RoundTripper } +// String implements Stringer +func (g *GogsDownloader) String() string { + return fmt.Sprintf("migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + +// ColorFormat provides a basic color format for a GogsDownloader +func (g *GogsDownloader) ColorFormat(s fmt.State) { + if g == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from gogs server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) +} + // SetContext set context func (g *GogsDownloader) SetContext(ctx context.Context) { g.ctx = ctx diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index de6d69ab7e361..06726f6d3667d 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -195,19 +195,25 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload return err } - // If the downloader is not a RepositoryRestorer then we need to recheck the CloneURL + // SECURITY: If the downloader is not a RepositoryRestorer then we need to recheck the CloneURL if _, ok := downloader.(*RepositoryRestorer); !ok { // Now the clone URL can be rewritten by the downloader so we must recheck if err := IsMigrateURLAllowed(repo.CloneURL, doer); err != nil { return err } - // And so can the original URL too so again we must recheck - if repo.OriginalURL != "" { - if err := IsMigrateURLAllowed(repo.OriginalURL, doer); err != nil { - return err + // SECURITY: Ensure that we haven't been redirected from an external to a local filesystem + // Now we know all of these must parse + cloneAddrURL, _ := url.Parse(opts.CloneAddr) + cloneURL, _ := url.Parse(repo.CloneURL) + + if cloneURL.Scheme == "file" || cloneURL.Scheme == "" { + if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" { + return fmt.Errorf("repo info has changed from external to local filesystem") } } + + // We don't actually need to check the OriginalURL as it isn't used anywhere } log.Trace("migrating git data from %s", repo.CloneURL) diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index d4b30939ce954..9c4f613c5f4d4 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -110,6 +110,20 @@ func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, passwo return downloader } +// String implements Stringer +func (d *OneDevDownloader) String() string { + return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName) +} + +// ColorFormat provides a basic color format for a OneDevDownloader +func (d *OneDevDownloader) ColorFormat(s fmt.State) { + if d == nil { + log.ColorFprintf(s, "") + return + } + log.ColorFprintf(s, "migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName) +} + func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error { u, err := d.baseURL.Parse(endpoint) if err != nil { @@ -542,6 +556,9 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque ForeignIndex: pr.ID, Context: onedevIssueContext{IsPullRequest: true}, }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) } return pullRequests, len(pullRequests) == 0, nil diff --git a/services/migrations/restore.go b/services/migrations/restore.go index 8c9654a7e3c2b..c3fbcbb25fa88 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -243,6 +243,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) + CheckAndEnsureSafePR(pr, "", r) } return pulls, true, nil } diff --git a/services/release/release.go b/services/release/release.go index 6fa966de1f352..22efe2f6a528e 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -37,6 +37,9 @@ func createTag(gitRepo *git.Repository, rel *models.Release, msg string) (bool, if err != nil { return false, fmt.Errorf("GetProtectedTags: %v", err) } + + // Trim '--' prefix to prevent command line argument vulnerability. + rel.TagName = strings.TrimPrefix(rel.TagName, "--") isAllowed, err := git_model.IsUserAllowedToControlTag(protectedTags, rel.TagName, rel.PublisherID) if err != nil { return false, err @@ -52,8 +55,6 @@ func createTag(gitRepo *git.Repository, rel *models.Release, msg string) (bool, return false, fmt.Errorf("createTag::GetCommit[%v]: %v", rel.Target, err) } - // Trim '--' prefix to prevent command line argument vulnerability. - rel.TagName = strings.TrimPrefix(rel.TagName, "--") if len(msg) > 0 { if err = gitRepo.CreateAnnotatedTag(rel.TagName, msg, commit.ID.String()); err != nil { if strings.Contains(err.Error(), "is not a valid tag name") { @@ -308,7 +309,7 @@ func DeleteReleaseByID(ctx context.Context, id int64, doer *user_model.User, del } } - if stdout, _, err := git.NewCommand(ctx, "tag", "-d", rel.TagName). + if stdout, _, err := git.NewCommand(ctx, "tag", "-d", "--", rel.TagName). SetDescription(fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID)). RunStdString(&git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err) diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index ebd3eaf236ab9..ae43503bae125 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -57,6 +57,21 @@ func (ErrUnknownArchiveFormat) Is(err error) bool { return ok } +// RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found. +type RepoRefNotFoundError struct { + RefName string +} + +// Error implements error. +func (e RepoRefNotFoundError) Error() string { + return fmt.Sprintf("unrecognized repository reference: %s", e.RefName) +} + +func (e RepoRefNotFoundError) Is(err error) bool { + _, ok := err.(RepoRefNotFoundError) + return ok +} + // NewRequest creates an archival request, based on the URI. The // resulting ArchiveRequest is suitable for being passed to ArchiveRepository() // if it's determined that the request still needs to be satisfied. @@ -103,7 +118,7 @@ func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest } } } else { - return nil, fmt.Errorf("Unknow ref %s type", r.refName) + return nil, RepoRefNotFoundError{RefName: r.refName} } return r, nil @@ -115,6 +130,49 @@ func (aReq *ArchiveRequest) GetArchiveName() string { return strings.ReplaceAll(aReq.refName, "/", "-") + "." + aReq.Type.String() } +// Await awaits the completion of an ArchiveRequest. If the archive has +// already been prepared the method returns immediately. Otherwise an archiver +// process will be started and its completion awaited. On success the returned +// RepoArchiver may be used to download the archive. Note that even if the +// context is cancelled/times out a started archiver will still continue to run +// in the background. +func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver, error) { + archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("models.GetRepoArchiver: %v", err) + } + + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + // Archive already generated, we're done. + return archiver, nil + } + + if err := StartArchive(aReq); err != nil { + return nil, fmt.Errorf("archiver.StartArchive: %v", err) + } + + poll := time.NewTicker(time.Second * 1) + defer poll.Stop() + + for { + select { + case <-graceful.GetManager().HammerContext().Done(): + // System stopped. + return nil, graceful.GetManager().HammerContext().Err() + case <-ctx.Done(): + return nil, ctx.Err() + case <-poll.C: + archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("repo_model.GetRepoArchiver: %v", err) + } + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + return archiver, nil + } + } + } +} + func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) { txCtx, committer, err := db.TxContext() if err != nil { @@ -147,11 +205,7 @@ func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) { } } - rPath, err := archiver.RelativePath() - if err != nil { - return nil, err - } - + rPath := archiver.RelativePath() _, err = storage.RepoArchives.Stat(rPath) if err == nil { if archiver.Status == repo_model.ArchiverGenerating { @@ -284,13 +338,10 @@ func StartArchive(request *ArchiveRequest) error { } func deleteOldRepoArchiver(ctx context.Context, archiver *repo_model.RepoArchiver) error { - p, err := archiver.RelativePath() - if err != nil { - return err - } if err := repo_model.DeleteRepoArchiver(ctx, archiver); err != nil { return err } + p := archiver.RelativePath() if err := storage.RepoArchives.Delete(p); err != nil { log.Error("delete repo archive file failed: %v", err) } diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 63ef828606e9c..25088d8b60474 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -4614,7 +4614,7 @@ }, { "type": "string", - "description": "The name of the commit/branch/tag. Default the repository’s default branch (usually master)", + "description": "The name of the commit/branch/tag, indicates which commit will be loaded to the webhook payload.", "name": "ref", "in": "query" } diff --git a/web_src/less/chroma/dark.less b/web_src/less/chroma/dark.less index 67531ec962411..4be9cf7912e0b 100644 --- a/web_src/less/chroma/dark.less +++ b/web_src/less/chroma/dark.less @@ -1,67 +1,70 @@ +.chroma .bp { color: #fabd2f; } /* NameBuiltinPseudo */ +.chroma .c { color: #777e94; } /* Comment */ +.chroma .c1 { color: #777e94; } /* CommentSingle */ +.chroma .ch { color: #777e94; } /* CommentHashbang */ +.chroma .cm { color: #777e94; } /* CommentMultiline */ +.chroma .cp { color: #8ec07c; } /* CommentPreproc */ +.chroma .cpf { color: #649bc4; } /* CommentPreprocFile */ +.chroma .cs { color: #9075cd; } /* CommentSpecial */ +.chroma .dl { color: #649bc4; } /* LiteralStringDelimiter */ +.chroma .gd { color: #ffffff; background-color: #5f3737; } /* GenericDeleted */ +.chroma .ge { color: #ddee30; } /* GenericEmph */ +.chroma .gh { color: #ffaa10; } /* GenericHeading */ +.chroma .gi { color: #ffffff; background-color: #3a523a; } /* GenericInserted */ +.chroma .go { color: #777e94; } /* GenericOutput */ +.chroma .gp { color: #ebdbb2; } /* GenericPrompt */ +.chroma .gr { color: #ff4433; } /* GenericError */ +.chroma .gs { color: #ebdbb2; } /* GenericStrong */ +.chroma .gt { color: #ff7540; } /* GenericTraceback */ +.chroma .gu { color: #b8bb26; } /* GenericSubheading */ .chroma .hl { background-color: #3f424d; } /* LineHighlight */ -.chroma .lnt { color: #7f7f7f; } /* LineNumbersTable */ -.chroma .ln { color: #7f7f7f; } /* LineNumbers */ -.chroma .k { color: #f63; } /* Keyword */ -.chroma .kc { color: #fa1; } /* KeywordConstant */ -.chroma .kd { color: #9daccc; } /* KeywordDeclaration */ -.chroma .kn { color: #fa1; } /* KeywordNamespace */ +.chroma .il { color: #649bc4; } /* LiteralNumberIntegerLong */ +.chroma .k { color: #ff7540; } /* Keyword */ +.chroma .kc { color: #649bc4; } /* KeywordConstant */ +.chroma .kd { color: #ff7540; } /* KeywordDeclaration */ +.chroma .kn { color: #ffaa10; } /* KeywordNamespace */ .chroma .kp { color: #5f8700; } /* KeywordPseudo */ -.chroma .kr { color: #f63; } /* KeywordReserved */ -.chroma .kt { color: #9daccc; } /* KeywordType */ -.chroma .na { color: #8a8a8a; } /* NameAttribute */ -.chroma .nb { color: #9daccc; } /* NameBuiltin */ -.chroma .bp { color: #9daccc; } /* NameBuiltinPseudo */ -.chroma .nc { color: #fa1; } /* NameClass */ -.chroma .no { color: #fa1; } /* NameConstant */ -.chroma .nd { color: #9daccc; } /* NameDecorator */ -.chroma .ni { color: #fa1; } /* NameEntity */ -.chroma .ne { color: #af8700; } /* NameException */ -.chroma .nf { color: #9daccc; } /* NameFunction */ -.chroma .nl { color: #fa1; } /* NameLabel */ -.chroma .nn { color: #fa1; } /* NameNamespace */ -.chroma .nx { color: #9daccc; } /* NameOther */ -.chroma .nt { color: #9daccc; } /* NameTag */ -.chroma .nv { color: #9daccc; } /* NameVariable */ -.chroma .vc { color: #f81; } /* NameVariableClass */ -.chroma .vg { color: #fa1; } /* NameVariableGlobal */ -.chroma .vi { color: #fa1; } /* NameVariableInstance */ -.chroma .s { color: #1af; } /* LiteralString */ -.chroma .sa { color: #1af; } /* LiteralStringAffix */ -.chroma .sb { color: #a0cc75; } /* LiteralStringBacktick */ -.chroma .sc { color: #1af; } /* LiteralStringChar */ -.chroma .dl { color: #1af; } /* LiteralStringDelimiter */ -.chroma .sd { color: #6a737d; } /* LiteralStringDoc */ -.chroma .s2 { color: #a0cc75; } /* LiteralStringDouble */ -.chroma .se { color: #f63; } /* LiteralStringEscape */ -.chroma .sh { color: #1af; } /* LiteralStringHeredoc */ -.chroma .si { color: #fa1; } /* LiteralStringInterpol */ -.chroma .sx { color: #fa1; } /* LiteralStringOther */ -.chroma .sr { color: #97c; } /* LiteralStringRegex */ -.chroma .s1 { color: #a0cc75; } /* LiteralStringSingle */ -.chroma .ss { color: #fa1; } /* LiteralStringSymbol */ -.chroma .m { color: #1af; } /* LiteralNumber */ -.chroma .mb { color: #1af; } /* LiteralNumberBin */ -.chroma .mf { color: #1af; } /* LiteralNumberFloat */ -.chroma .mh { color: #1af; } /* LiteralNumberHex */ -.chroma .mi { color: #1af; } /* LiteralNumberInteger */ -.chroma .il { color: #1af; } /* LiteralNumberIntegerLong */ -.chroma .mo { color: #1af; } /* LiteralNumberOct */ -.chroma .o { color: #f63; } /* Operator */ +.chroma .kr { color: #ff7540; } /* KeywordReserved */ +.chroma .kt { color: #fabd2f; } /* KeywordType */ +.chroma .ln { color: #7f8699; } /* LineNumbers */ +.chroma .lnt { color: #7f8699; } /* LineNumbersTable */ +.chroma .m { color: #649bc4; } /* LiteralNumber */ +.chroma .mb { color: #649bc4; } /* LiteralNumberBin */ +.chroma .mf { color: #649bc4; } /* LiteralNumberFloat */ +.chroma .mh { color: #649bc4; } /* LiteralNumberHex */ +.chroma .mi { color: #649bc4; } /* LiteralNumberInteger */ +.chroma .mo { color: #649bc4; } /* LiteralNumberOct */ +.chroma .n { color: #fabd2f; } /* Name */ +.chroma .na { color: #b8bb26; } /* NameAttribute */ +.chroma .nb { color: #fabd2f; } /* NameBuiltin */ +.chroma .nc { color: #ffaa10; } /* NameClass */ +.chroma .nd { color: #8ec07c; } /* NameDecorator */ +.chroma .ne { color: #ff7540; } /* NameException */ +.chroma .nf { color: #fabd2f; } /* NameFunction */ +.chroma .ni { color: #fabd2f; } /* NameEntity */ +.chroma .nl { color: #ff7540; } /* NameLabel */ +.chroma .nn { color: #ffaa10; } /* NameNamespace */ +.chroma .no { color: #649bc4; } /* NameConstant */ +.chroma .nt { color: #ff7540; } /* NameTag */ +.chroma .nv { color: #ebdbb2; } /* NameVariable */ +.chroma .nx { color: #b6bac5; } /* NameOther */ +.chroma .o { color: #ff7540; } /* Operator */ .chroma .ow { color: #5f8700; } /* OperatorWord */ -.chroma .c { color: #6a737d; } /* Comment */ -.chroma .ch { color: #6a737d; } /* CommentHashbang */ -.chroma .cm { color: #6a737d; } /* CommentMultiline */ -.chroma .c1 { color: #6a737d; } /* CommentSingle */ -.chroma .cs { color: #95ad; } /* CommentSpecial */ -.chroma .cp { color: #fc6; } /* CommentPreproc */ -.chroma .cpf { color: #03dfff; } /* CommentPreprocFile */ -.chroma .gd { color: #fff; background-color: #5f3737; } /* GenericDeleted */ -.chroma .ge { color: #ef5; } /* GenericEmph */ -.chroma .gr { color: #f33; } /* GenericError */ -.chroma .gh { color: #fa1; } /* GenericHeading */ -.chroma .gi { color: #fff; background-color: #3a523a; } /* GenericInserted */ -.chroma .go { color: #888888; } /* GenericOutput */ -.chroma .gp { color: #555555; } /* GenericPrompt */ -.chroma .gu { color: #9daccc; } /* GenericSubheading */ -.chroma .gt { color: #f63; } /* GenericTraceback */ -.chroma .w { color: #bbbbbb; } /* TextWhitespace */ +.chroma .p { color: #d2d4db; } /* Punctuation */ +.chroma .s { color: #b8bb26; } /* LiteralString */ +.chroma .s1 { color: #b8bb26; } /* LiteralStringSingle */ +.chroma .s2 { color: #b8bb26; } /* LiteralStringDouble */ +.chroma .sa { color: #649bc4; } /* LiteralStringAffix */ +.chroma .sb { color: #b8bb26; } /* LiteralStringBacktick */ +.chroma .sc { color: #649bc4; } /* LiteralStringChar */ +.chroma .sd { color: #777e94; } /* LiteralStringDoc */ +.chroma .se { color: #ff7540; } /* LiteralStringEscape */ +.chroma .sh { color: #649bc4; } /* LiteralStringHeredoc */ +.chroma .si { color: #ffaa10; } /* LiteralStringInterpol */ +.chroma .sr { color: #9075cd; } /* LiteralStringRegex */ +.chroma .ss { color: #ff7540; } /* LiteralStringSymbol */ +.chroma .sx { color: #ffaa10; } /* LiteralStringOther */ +.chroma .vc { color: #ff7540; } /* NameVariableClass */ +.chroma .vg { color: #ffaa10; } /* NameVariableGlobal */ +.chroma .vi { color: #ffaa10; } /* NameVariableInstance */ +.chroma .w { color: #7f8699; } /* TextWhitespace */ diff --git a/web_src/less/chroma/light.less b/web_src/less/chroma/light.less index 9728800765b43..2e811844c2ddd 100644 --- a/web_src/less/chroma/light.less +++ b/web_src/less/chroma/light.less @@ -1,6 +1,23 @@ +.chroma .bp { color: #999999; } /* NameBuiltinPseudo */ +.chroma .c { color: #6a737d; } /* Comment */ +.chroma .c1 { color: #6a737d; } /* CommentSingle */ +.chroma .ch { color: #6a737d; } /* CommentHashbang */ +.chroma .cm { color: #999988; } /* CommentMultiline */ +.chroma .cp { color: #109295; } /* CommentPreproc */ +.chroma .cpf { color: #4c4dbc; } /* CommentPreprocFile */ +.chroma .cs { color: #999999; } /* CommentSpecial */ +.chroma .dl { color: #106303; } /* LiteralStringDelimiter */ +.chroma .gd { color: #000000; background-color: #ffdddd; } /* GenericDeleted */ +.chroma .ge { color: #000000; } /* GenericEmph */ +.chroma .gh { color: #999999; } /* GenericHeading */ +.chroma .gi { color: #000000; background-color: #ddffdd; } /* GenericInserted */ +.chroma .go { color: #888888; } /* GenericOutput */ +.chroma .gp { color: #555555; } /* GenericPrompt */ +.chroma .gr { color: #aa0000; } /* GenericError */ +.chroma .gt { color: #aa0000; } /* GenericTraceback */ +.chroma .gu { color: #aaaaaa; } /* GenericSubheading */ .chroma .hl { background-color: #e5e5e5; } /* LineHighlight */ -.chroma .lnt { color: #7f7f7f; } /* LineNumbersTable */ -.chroma .ln { color: #7f7f7f; } /* LineNumbers */ +.chroma .il { color: #009999; } /* LiteralNumberIntegerLong */ .chroma .k { color: #d73a49; } /* Keyword */ .chroma .kc { color: #d73a49; } /* KeywordConstant */ .chroma .kd { color: #d73a49; } /* KeywordDeclaration */ @@ -8,60 +25,43 @@ .chroma .kp { color: #d73a49; } /* KeywordPseudo */ .chroma .kr { color: #d73a49; } /* KeywordReserved */ .chroma .kt { color: #445588; } /* KeywordType */ +.chroma .ln { color: #7f7f7f; } /* LineNumbers */ +.chroma .lnt { color: #7f7f7f; } /* LineNumbersTable */ +.chroma .m { color: #009999; } /* LiteralNumber */ +.chroma .mb { color: #009999; } /* LiteralNumberBin */ +.chroma .mf { color: #009999; } /* LiteralNumberFloat */ +.chroma .mh { color: #009999; } /* LiteralNumberHex */ +.chroma .mi { color: #009999; } /* LiteralNumberInteger */ +.chroma .mo { color: #009999; } /* LiteralNumberOct */ .chroma .na { color: #d73a49; } /* NameAttribute */ .chroma .nb { color: #005cc5; } /* NameBuiltin */ -.chroma .bp { color: #999999; } /* NameBuiltinPseudo */ .chroma .nc { color: #445588; } /* NameClass */ -.chroma .no { color: #008080; } /* NameConstant */ .chroma .nd { color: #3c5d5d; } /* NameDecorator */ -.chroma .ni { color: #6f42c1; } /* NameEntity */ .chroma .ne { color: #990000; } /* NameException */ .chroma .nf { color: #005cc5; } /* NameFunction */ +.chroma .ni { color: #6f42c1; } /* NameEntity */ .chroma .nl { color: #990000; } /* NameLabel */ .chroma .nn { color: #555555; } /* NameNamespace */ -.chroma .nx { color: #24292e; } /* NameOther */ +.chroma .no { color: #008080; } /* NameConstant */ .chroma .nt { color: #22863a; } /* NameTag */ .chroma .nv { color: #008080; } /* NameVariable */ -.chroma .vc { color: #008080; } /* NameVariableClass */ -.chroma .vg { color: #008080; } /* NameVariableGlobal */ -.chroma .vi { color: #008080; } /* NameVariableInstance */ +.chroma .nx { color: #24292e; } /* NameOther */ +.chroma .o { color: #d73a49; } /* Operator */ +.chroma .ow { color: #d73a49; } /* OperatorWord */ .chroma .s { color: #106303; } /* LiteralString */ +.chroma .s1 { color: #cc7a00; } /* LiteralStringSingle */ +.chroma .s2 { color: #106303; } /* LiteralStringDouble */ .chroma .sa { color: #106303; } /* LiteralStringAffix */ .chroma .sb { color: #106303; } /* LiteralStringBacktick */ .chroma .sc { color: #106303; } /* LiteralStringChar */ -.chroma .dl { color: #106303; } /* LiteralStringDelimiter */ .chroma .sd { color: #106303; } /* LiteralStringDoc */ -.chroma .s2 { color: #106303; } /* LiteralStringDouble */ .chroma .se { color: #106303; } /* LiteralStringEscape */ .chroma .sh { color: #106303; } /* LiteralStringHeredoc */ .chroma .si { color: #106303; } /* LiteralStringInterpol */ -.chroma .sx { color: #106303; } /* LiteralStringOther */ .chroma .sr { color: #22863a; } /* LiteralStringRegex */ -.chroma .s1 { color: #cc7a00; } /* LiteralStringSingle */ .chroma .ss { color: #106303; } /* LiteralStringSymbol */ -.chroma .m { color: #009999; } /* LiteralNumber */ -.chroma .mb { color: #009999; } /* LiteralNumberBin */ -.chroma .mf { color: #009999; } /* LiteralNumberFloat */ -.chroma .mh { color: #009999; } /* LiteralNumberHex */ -.chroma .mi { color: #009999; } /* LiteralNumberInteger */ -.chroma .il { color: #009999; } /* LiteralNumberIntegerLong */ -.chroma .mo { color: #009999; } /* LiteralNumberOct */ -.chroma .o { color: #d73a49; } /* Operator */ -.chroma .ow { color: #d73a49; } /* OperatorWord */ -.chroma .c { color: #6a737d; } /* Comment */ -.chroma .ch { color: #6a737d; } /* CommentHashbang */ -.chroma .cm { color: #999988; } /* CommentMultiline */ -.chroma .c1 { color: #6a737d; } /* CommentSingle */ -.chroma .cs { color: #999999; } /* CommentSpecial */ -.chroma .cp { color: #109295; } /* CommentPreproc */ -.chroma .cpf { color: #4c4dbc; } /* CommentPreprocFile */ -.chroma .gd { color: #000000; background-color: #ffdddd; } /* GenericDeleted */ -.chroma .ge { color: #000000; } /* GenericEmph */ -.chroma .gr { color: #aa0000; } /* GenericError */ -.chroma .gh { color: #999999; } /* GenericHeading */ -.chroma .gi { color: #000000; background-color: #ddffdd; } /* GenericInserted */ -.chroma .go { color: #888888; } /* GenericOutput */ -.chroma .gp { color: #555555; } /* GenericPrompt */ -.chroma .gu { color: #aaaaaa; } /* GenericSubheading */ -.chroma .gt { color: #aa0000; } /* GenericTraceback */ +.chroma .sx { color: #106303; } /* LiteralStringOther */ +.chroma .vc { color: #008080; } /* NameVariableClass */ +.chroma .vg { color: #008080; } /* NameVariableGlobal */ +.chroma .vi { color: #008080; } /* NameVariableInstance */ .chroma .w { color: #bbbbbb; } /* TextWhitespace */