From 7a083b920d19570f7bcc4bac04dca09d21438817 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 14:55:07 +0800 Subject: [PATCH 01/23] add black list and white list support for migrating repositories --- custom/conf/app.ini.sample | 8 +++ .../doc/advanced/config-cheat-sheet.en-us.md | 6 +++ .../doc/advanced/config-cheat-sheet.zh-cn.md | 6 +++ modules/matchlist/matchlist.go | 43 +++++++++++++++ modules/migrations/migrate.go | 52 +++++++++++++++++++ modules/migrations/migrate_test.go | 38 ++++++++++++++ modules/setting/migrate.go | 25 +++++++++ routers/init.go | 5 ++ vendor/modules.txt | 1 + 9 files changed, 184 insertions(+) create mode 100644 modules/matchlist/matchlist.go create mode 100644 modules/migrations/migrate_test.go create mode 100644 modules/setting/migrate.go diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index a2ac7248e770b..962bfa0f0dba7 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -803,3 +803,11 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[migration] +; Whitelist for migrating, default is blank. Blank means everything will be allowed. +; Multiple domains could be separated by commas. +WHITELISTED_DOMAINS = +; Blacklist for migrating, default is blank. Multiple domains could be separated by commas. +; When WHITELISTED_DOMAINS is not blank, this option will be ignored. +BLACKLISTED_DOMAINS = \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 10c6110f373f2..628a1fb57e865 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -511,9 +511,15 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. ## Time (`time`) + - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia +## Migraions (`migration`) + +- `WHITELISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLACKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `WHITELISTED_DOMAINS` is not blank, this option will be ignored. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index b4b9e4e3a92a6..c9fba9fa07e31 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -240,9 +240,15 @@ IS_INPUT_FILE = false - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 ## Time (`time`) + - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai +## Migraions (`migration`) + +- `WHITELISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLACKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `WHITELISTED_DOMAINS` 不为空，此选项将会被忽略。 + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go new file mode 100644 index 0000000000000..b417e38099c99 --- /dev/null +++ b/modules/matchlist/matchlist.go @@ -0,0 +1,43 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package matchlist + +import ( + "github.com/gobwas/glob" +) + +// Matchlist represents a black or white list +type Matchlist struct { + rules []string + ruleGlobs []glob.Glob +} + +// NewMatchlist creates a new black or white list +func NewMatchlist(rules ...string) (*Matchlist, error) { + list := Matchlist{ + rules: rules, + ruleGlobs: make([]glob.Glob, 0, len(rules)), + } + + for _, rule := range list.rules { + rg, err := glob.Compile(rule) + if err != nil { + return nil, err + } + list.ruleGlobs = append(list.ruleGlobs, rg) + } + + return &list, nil +} + +// Match will matches +func (b *Matchlist) Match(u string) bool { + for _, r := range b.ruleGlobs { + if r.Match(u) { + return true + } + } + return false +} \ No newline at end of file diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 27782cb94034e..5fe1c027eec43 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,9 +6,15 @@ package migrations import ( + "fmt" + "net/url" + "strings" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/matchlist" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" ) // MigrateOptions is equal to base.MigrateOptions @@ -23,8 +29,34 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { factories = append(factories, factory) } +func isMigrateURLAllowed(remoteURL string) (bool, error) { + u, err := url.Parse(remoteURL) + if err != nil { + return false, err + } + + if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { + if len(setting.Migration.WhitelistedDomains) > 0 { + if !whitelist.Match(u.Host) { + return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + } + } else { + if blacklist.Match(u.Host) { + return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + } + } + } + + return true, nil +} + // MigrateRepository migrate repository according MigrateOptions func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { + allowed, err := isMigrateURLAllowed(opts.RemoteURL) + if !allowed { + return nil, err + } + var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) @@ -250,3 +282,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return nil } + +var ( + whitelist *matchlist.Matchlist + blacklist *matchlist.Matchlist +) + +// Init migrations service +func Init() error { + var err error + whitelist, err = matchlist.NewMatchlist(setting.Migration.WhitelistedDomains...) + if err != nil { + return fmt.Errorf("Init migration whitelist domains failed: %v", err) + } + + blacklist, err = matchlist.NewMatchlist(setting.Migration.BlacklistedDomains...) + if err != nil { + return fmt.Errorf("Init migration blacklist domains failed: %v", err) + } + return nil +} \ No newline at end of file diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go new file mode 100644 index 0000000000000..275f48d1a949c --- /dev/null +++ b/modules/migrations/migrate_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestMigrateWhiteBlacklist(t *testing.T) { + setting.Migration.WhitelistedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.False(t, allowed) + assert.Error(t, err) + + allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.True(t, allowed) + assert.NoError(t, err) + + setting.Migration.WhitelistedDomains = []string{} + setting.Migration.BlacklistedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.True(t, allowed) + assert.NoError(t, err) + + allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.False(t, allowed) + assert.Error(t, err) +} \ No newline at end of file diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go new file mode 100644 index 0000000000000..eb634ef23fdce --- /dev/null +++ b/modules/setting/migrate.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "fmt" +) + +var Migration = struct { + WhitelistedDomains []string + BlacklistedDomains []string +} { + WhitelistedDomains: []string{}, + BlacklistedDomains: []string{}, +} + +// InitMigrationConfig represents load migration configurations +func InitMigrationConfig() error { + if err := Cfg.Section("migration").MapTo(&Migration); err != nil { + return fmt.Errorf("Failed to map Migration settings: %v", err) + } + return nil +} \ No newline at end of file diff --git a/routers/init.go b/routers/init.go index fdf90904ce759..b641de8560394 100644 --- a/routers/init.go +++ b/routers/init.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/mailer" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" + repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" @@ -101,6 +102,10 @@ func GlobalInit() { models.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() + + if err := repo_migrations.Init(); err != nil { + log.Fatal("Failed to initialize migrations: %v", err) + } } if setting.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/vendor/modules.txt b/vendor/modules.txt index 27dc32d705be2..498377dd67749 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -524,6 +524,7 @@ golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/module # google.golang.org/appengine v1.6.2 +# google.golang.org/appengine v1.6.1 google.golang.org/appengine/cloudsql google.golang.org/appengine/urlfetch google.golang.org/appengine/internal From 3f2b34ba86e4ba4f767e5369b837e38d3ebd7fb0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:11:27 +0800 Subject: [PATCH 02/23] fix fmt --- modules/matchlist/matchlist.go | 6 +++--- modules/migrations/migrate.go | 4 ++-- modules/migrations/migrate_test.go | 2 +- modules/setting/migrate.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go index b417e38099c99..656d8fc99174c 100644 --- a/modules/matchlist/matchlist.go +++ b/modules/matchlist/matchlist.go @@ -10,14 +10,14 @@ import ( // Matchlist represents a black or white list type Matchlist struct { - rules []string + rules []string ruleGlobs []glob.Glob } // NewMatchlist creates a new black or white list func NewMatchlist(rules ...string) (*Matchlist, error) { list := Matchlist{ - rules: rules, + rules: rules, ruleGlobs: make([]glob.Glob, 0, len(rules)), } @@ -40,4 +40,4 @@ func (b *Matchlist) Match(u string) bool { } } return false -} \ No newline at end of file +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 5fe1c027eec43..67f3e308d27d3 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -56,7 +56,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt if !allowed { return nil, err } - + var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) @@ -301,4 +301,4 @@ func Init() error { return fmt.Errorf("Init migration blacklist domains failed: %v", err) } return nil -} \ No newline at end of file +} diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index 275f48d1a949c..534625018913b 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -35,4 +35,4 @@ func TestMigrateWhiteBlacklist(t *testing.T) { allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") assert.False(t, allowed) assert.Error(t, err) -} \ No newline at end of file +} diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go index eb634ef23fdce..d26683692fe11 100644 --- a/modules/setting/migrate.go +++ b/modules/setting/migrate.go @@ -11,7 +11,7 @@ import ( var Migration = struct { WhitelistedDomains []string BlacklistedDomains []string -} { +}{ WhitelistedDomains: []string{}, BlacklistedDomains: []string{}, } @@ -22,4 +22,4 @@ func InitMigrationConfig() error { return fmt.Errorf("Failed to map Migration settings: %v", err) } return nil -} \ No newline at end of file +} From ad5a226cce1da48b2d5df22e51b6391f5e07dc93 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:25:10 +0800 Subject: [PATCH 03/23] fix lint --- modules/setting/migrate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go index d26683692fe11..657bb969f590b 100644 --- a/modules/setting/migrate.go +++ b/modules/setting/migrate.go @@ -8,6 +8,7 @@ import ( "fmt" ) +// Migration represents migrations' settings var Migration = struct { WhitelistedDomains []string BlacklistedDomains []string From 7280964e5f14fed5f43effffec2d1339a9ddc4de Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:43:37 +0800 Subject: [PATCH 04/23] fix vendor --- vendor/modules.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index 498377dd67749..ab99c226856cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -444,6 +444,7 @@ go.mongodb.org/mongo-driver/bson/bsoncodec go.mongodb.org/mongo-driver/bson/bsonrw go.mongodb.org/mongo-driver/x/bsonx/bsoncore # golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 +# golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 golang.org/x/crypto/acme/autocert golang.org/x/crypto/argon2 golang.org/x/crypto/bcrypt From cdff51c8946a9f6b06316c871d769b6cd4fef114 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 7 Sep 2019 11:41:58 +0800 Subject: [PATCH 05/23] fix modules.txt --- vendor/modules.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/vendor/modules.txt b/vendor/modules.txt index ab99c226856cc..27dc32d705be2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -444,7 +444,6 @@ go.mongodb.org/mongo-driver/bson/bsoncodec go.mongodb.org/mongo-driver/bson/bsonrw go.mongodb.org/mongo-driver/x/bsonx/bsoncore # golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 -# golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 golang.org/x/crypto/acme/autocert golang.org/x/crypto/argon2 golang.org/x/crypto/bcrypt @@ -525,7 +524,6 @@ golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/module # google.golang.org/appengine v1.6.2 -# google.golang.org/appengine v1.6.1 google.golang.org/appengine/cloudsql google.golang.org/appengine/urlfetch google.golang.org/appengine/internal From ec4dbc85946e99406a4fae503cafd21a427a64c8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 17 Nov 2020 23:50:11 +0100 Subject: [PATCH 06/23] clean diff --- .../content/doc/advanced/config-cheat-sheet.en-us.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 59ba303a6e377..befa50e9a1720 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -240,7 +240,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `SSH_PORT`: **22**: SSH port displayed in clone URL. - `SSH_LISTEN_HOST`: **0.0.0.0**: Listen address for the built-in SSH server. - `SSH_LISTEN_PORT`: **%(SSH\_PORT)s**: Port for the built-in SSH server. -- `SSH_ROOT_PATH`: **~/.ssh**: Root path of SSH directory. +- `SSH_ROOT_PATH`: **~/.ssh**: Root path of SSH directory. - `SSH_CREATE_AUTHORIZED_KEYS_FILE`: **true**: Gitea will create a authorized_keys file by default when it is not using the internal ssh server. If you intend to use the AuthorizedKeysCommand functionality then you should turn this off. - `SSH_AUTHORIZED_KEYS_BACKUP`: **true**: Enable SSH Authorized Key Backup when rewriting all keys, default is true. - `SSH_TRUSTED_USER_CA_KEYS`: **\**: Specifies the public keys of certificate authorities that are trusted to sign user certificates for authentication. Multiple keys should be comma separated. E.g.`ssh- ` or `ssh- , ssh- `. For more information see `TrustedUserCAKeys` in the sshd config man pages. When empty no file will be created and `SSH_AUTHORIZED_PRINCIPALS_ALLOW` will default to `off`. @@ -294,7 +294,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `USER`: **root**: Database username. - `PASSWD`: **\**: Database user password. Use \`your password\` or """your password""" for quoting if you use special characters in the password. - `SCHEMA`: **\**: For PostgreSQL only, schema to use if different from "public". The schema must exist beforehand, - the user must have creation privileges on it, and the user search path must be set to the look into the schema first + the user must have creation privileges on it, and the user search path must be set to the look into the schema first (e.g. `ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;`). - `SSL_MODE`: **disable**: SSL/TLS encryption mode for connecting to the database. This option is only applied for PostgreSQL and MySQL. - Valid values for MySQL: @@ -318,7 +318,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `MAX_OPEN_CONNS` **0**: Database maximum open connections - default is 0, meaning there is no limit. - `MAX_IDLE_CONNS` **2**: Max idle database connections on connnection pool, default is 2 - this will be capped to `MAX_OPEN_CONNS`. - `CONN_MAX_LIFETIME` **0 or 3s**: Sets the maximum amount of time a DB connection may be reused - default is 0, meaning there is no limit (except on MySQL where it is 3s - see #6804 & #7071). - + Please see #8540 & #8273 for further discussion of the appropriate values for `MAX_OPEN_CONNS`, `MAX_IDLE_CONNS` & `CONN_MAX_LIFETIME` and their relation to port exhaustion. @@ -465,7 +465,7 @@ relation to port exhaustion. - `DEFAULT_ORG_VISIBILITY`: **public**: Set default visibility mode for organisations, either "public", "limited" or "private". - `DEFAULT_ORG_MEMBER_VISIBLE`: **false** True will make the membership of the users visible when added to the organisation. - `ALLOW_ONLY_EXTERNAL_REGISTRATION`: **false** Set to true to force registration only using third-party services. -- `NO_REPLY_ADDRESS`: **DOMAIN** Default value for the domain part of the user's email address in the git log if he has set KeepEmailPrivate to true. +- `NO_REPLY_ADDRESS`: **DOMAIN** Default value for the domain part of the user's email address in the git log if he has set KeepEmailPrivate to true. The user's email will be replaced with a concatenation of the user name in lower case, "@" and NO_REPLY_ADDRESS. ## SSH Minimum Key Sizes (`ssh.minimum_key_sizes`) @@ -493,7 +493,7 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type - `HELO_HOSTNAME`: **\**: Custom hostname for HELO operation. - `HOST`: **\**: SMTP mail host address and port (example: smtp.gitea.io:587). - Using opportunistic TLS via STARTTLS on port 587 is recommended per RFC 6409. -- `IS_TLS_ENABLED` : **false** : Forcibly use TLS to connect even if not on a default SMTPS port. +- `IS_TLS_ENABLED` : **false** : Forcibly use TLS to connect even if not on a default SMTPS port. - Note, if the port ends with `465` SMTPS/SMTP over TLS will be used despite this setting. - Otherwise if `IS_TLS_ENABLED=false` and the server supports `STARTTLS` this will be used. Thus if `STARTTLS` is preferred you should set `IS_TLS_ENABLED=false`. - `FROM`: **\**: Mail from address, RFC 5322. This can be just an email address, or @@ -855,7 +855,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf ## LFS (`lfs`) Storage configuration for lfs data. It will be derived from default `[storage]` or -`[storage.xxx]` when set `STORAGE_TYPE` to `xxx`. When derived, the default of `PATH` +`[storage.xxx]` when set `STORAGE_TYPE` to `xxx`. When derived, the default of `PATH` is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`. - `STORAGE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service or other name defined with `[storage.xxx]` From 770dab6212ea76b5c265672e530ddcf7ecc5e63e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 17 Nov 2020 23:55:02 +0100 Subject: [PATCH 07/23] specify log message --- routers/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/init.go b/routers/init.go index 88c6712da5984..6434fa89ba560 100644 --- a/routers/init.go +++ b/routers/init.go @@ -203,7 +203,7 @@ func GlobalInit(ctx context.Context) { log.Fatal("Failed to initialize task scheduler: %v", err) } if err := repo_migrations.Init(); err != nil { - log.Fatal("Failed to initialize migrations: %v", err) + log.Fatal("Failed to initialize repository migrations: %v", err) } eventsource.GetManager().Init() From 05e45bf5a44da8f8be4dcbfe196dabeac17ebf66 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 18 Nov 2020 00:09:10 +0100 Subject: [PATCH 08/23] use blocklist/allowlist --- custom/conf/app.example.ini | 4 ++-- .../doc/advanced/config-cheat-sheet.en-us.md | 4 ++-- .../doc/advanced/config-cheat-sheet.zh-cn.md | 4 ++-- modules/migrations/migrate.go | 18 +++++++++--------- modules/migrations/migrate_test.go | 8 ++++---- modules/setting/migrate.go | 8 ++++---- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 3887b7d9330a8..08a93329f1bbf 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1216,7 +1216,7 @@ STORAGE_TYPE = local [migration] ; Whitelist for migrating, default is blank. Blank means everything will be allowed. ; Multiple domains could be separated by commas. -WHITELISTED_DOMAINS = +ALLOWLISTED_DOMAINS = ; Blacklist for migrating, default is blank. Multiple domains could be separated by commas. ; When WHITELISTED_DOMAINS is not blank, this option will be ignored. -BLACKLISTED_DOMAINS = +BLOCKLISTED_DOMAINS = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index befa50e9a1720..26ee0e3403d15 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -904,8 +904,8 @@ And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`. ## Migraions (`migration`) -- `WHITELISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. -- `BLACKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `WHITELISTED_DOMAINS` is not blank, this option will be ignored. +- `ALLOWLISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. ## Other (`other`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 70c0bac7c94a0..7fff6622d5251 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -365,8 +365,8 @@ MINIO_USE_SSL = false ## Migraions (`migration`) -- `WHITELISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 -- `BLACKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `WHITELISTED_DOMAINS` 不为空，此选项将会被忽略。 +- `ALLOWLISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLOCKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 ## Other (`other`) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 2935efc580f57..0ff67d70cd488 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -37,12 +37,12 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { } if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { - if len(setting.Migration.WhitelistedDomains) > 0 { - if !whitelist.Match(u.Host) { + if len(setting.Migration.AllowlistedDomains) > 0 { + if !allowlist.Match(u.Host) { return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) } } else { - if blacklist.Match(u.Host) { + if blocklist.Match(u.Host) { return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) } } @@ -338,21 +338,21 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } var ( - whitelist *matchlist.Matchlist - blacklist *matchlist.Matchlist + allowlist *matchlist.Matchlist + blocklist *matchlist.Matchlist ) // Init migrations service func Init() error { var err error - whitelist, err = matchlist.NewMatchlist(setting.Migration.WhitelistedDomains...) + allowlist, err = matchlist.NewMatchlist(setting.Migration.AllowlistedDomains...) if err != nil { - return fmt.Errorf("Init migration whitelist domains failed: %v", err) + return fmt.Errorf("init migration allowlist domains failed: %v", err) } - blacklist, err = matchlist.NewMatchlist(setting.Migration.BlacklistedDomains...) + blocklist, err = matchlist.NewMatchlist(setting.Migration.BlocklistedDomains...) if err != nil { - return fmt.Errorf("Init migration blacklist domains failed: %v", err) + return fmt.Errorf("init migration blocklist domains failed: %v", err) } return nil } diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index 534625018913b..6cb547db5295a 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMigrateWhiteBlacklist(t *testing.T) { - setting.Migration.WhitelistedDomains = []string{"github.com"} +func TestMigrateWhiteBlocklist(t *testing.T) { + setting.Migration.AllowlistedDomains = []string{"github.com"} assert.NoError(t, Init()) allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") @@ -24,8 +24,8 @@ func TestMigrateWhiteBlacklist(t *testing.T) { assert.True(t, allowed) assert.NoError(t, err) - setting.Migration.WhitelistedDomains = []string{} - setting.Migration.BlacklistedDomains = []string{"github.com"} + setting.Migration.AllowlistedDomains = []string{} + setting.Migration.BlocklistedDomains = []string{"github.com"} assert.NoError(t, Init()) allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go index 657bb969f590b..50bc98632fc25 100644 --- a/modules/setting/migrate.go +++ b/modules/setting/migrate.go @@ -10,11 +10,11 @@ import ( // Migration represents migrations' settings var Migration = struct { - WhitelistedDomains []string - BlacklistedDomains []string + AllowlistedDomains []string + BlocklistedDomains []string }{ - WhitelistedDomains: []string{}, - BlacklistedDomains: []string{}, + AllowlistedDomains: []string{}, + BlocklistedDomains: []string{}, } // InitMigrationConfig represents load migration configurations From a0787365badc89b3bee99a288ec3f35ea4cc2061 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 18 Nov 2020 00:16:12 +0100 Subject: [PATCH 09/23] allways use lowercase to match url --- modules/matchlist/matchlist.go | 5 +++++ modules/migrations/migrate.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go index 656d8fc99174c..de54882c9fb48 100644 --- a/modules/matchlist/matchlist.go +++ b/modules/matchlist/matchlist.go @@ -5,6 +5,8 @@ package matchlist import ( + "strings" + "github.com/gobwas/glob" ) @@ -16,6 +18,9 @@ type Matchlist struct { // NewMatchlist creates a new black or white list func NewMatchlist(rules ...string) (*Matchlist, error) { + for i := range rules { + rules[i] = strings.ToLower(rules[i]) + } list := Matchlist{ rules: rules, ruleGlobs: make([]glob.Glob, 0, len(rules)), diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 0ff67d70cd488..94bc84233265c 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -31,7 +31,7 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { } func isMigrateURLAllowed(remoteURL string) (bool, error) { - u, err := url.Parse(remoteURL) + u, err := url.Parse(strings.ToLower(remoteURL)) if err != nil { return false, err } From dec70f1a69b004c58dcdc5be513130c404b691c0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 18 Nov 2020 00:20:31 +0100 Subject: [PATCH 10/23] Apply allow/block --- custom/conf/app.example.ini | 6 +++--- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 4 ++-- modules/matchlist/matchlist.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 08a93329f1bbf..ebe604201a3e5 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1214,9 +1214,9 @@ STORAGE_TYPE = local ;MINIO_USE_SSL = false [migration] -; Whitelist for migrating, default is blank. Blank means everything will be allowed. +; Allowlist for migrating, default is blank. Blank means everything will be allowed. ; Multiple domains could be separated by commas. ALLOWLISTED_DOMAINS = -; Blacklist for migrating, default is blank. Multiple domains could be separated by commas. -; When WHITELISTED_DOMAINS is not blank, this option will be ignored. +; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. +; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. BLOCKLISTED_DOMAINS = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 26ee0e3403d15..7124836f98f2f 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -904,8 +904,8 @@ And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`. ## Migraions (`migration`) -- `ALLOWLISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. -- `BLOCKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. +- `ALLOWLISTED_DOMAINS`: ****: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKLISTED_DOMAINS`: ****: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. ## Other (`other`) diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go index de54882c9fb48..666f0d8e00973 100644 --- a/modules/matchlist/matchlist.go +++ b/modules/matchlist/matchlist.go @@ -10,13 +10,13 @@ import ( "github.com/gobwas/glob" ) -// Matchlist represents a black or white list +// Matchlist represents a block or allow list type Matchlist struct { rules []string ruleGlobs []glob.Glob } -// NewMatchlist creates a new black or white list +// NewMatchlist creates a new block or allow list func NewMatchlist(rules ...string) (*Matchlist, error) { for i := range rules { rules[i] = strings.ToLower(rules[i]) From 840fc854fe1f91a457c4f41ce86903b0ccded914 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 21 Nov 2020 18:42:42 +0100 Subject: [PATCH 11/23] Settings: use existing "migrations" section --- custom/conf/app.example.ini | 14 +++++----- .../doc/advanced/config-cheat-sheet.en-us.md | 7 ++--- .../doc/advanced/config-cheat-sheet.zh-cn.md | 7 ++--- modules/migrations/migrate.go | 6 ++--- modules/migrations/migrate_test.go | 6 ++--- modules/setting/migrate.go | 26 ------------------- modules/setting/migrations.go | 9 +++++-- 7 files changed, 23 insertions(+), 52 deletions(-) delete mode 100644 modules/setting/migrate.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index f72d2673fd995..33c8144d054ed 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1187,6 +1187,12 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" MAX_ATTEMPTS = 3 ; Backoff time per http/https request retry (seconds) RETRY_BACKOFF = 3 +; Allowlist for migrating, default is blank. Blank means everything will be allowed. +; Multiple domains could be separated by commas. +ALLOWLISTED_DOMAINS = +; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. +; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. +BLOCKLISTED_DOMAINS = ; default storage for attachments, lfs and avatars [storage] @@ -1212,11 +1218,3 @@ STORAGE_TYPE = local ;MINIO_LOCATION = us-east-1 ; Minio enabled ssl only available when STORAGE_TYPE is `minio` ;MINIO_USE_SSL = false - -[migration] -; Allowlist for migrating, default is blank. Blank means everything will be allowed. -; Multiple domains could be separated by commas. -ALLOWLISTED_DOMAINS = -; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. -; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. -BLOCKLISTED_DOMAINS = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index d4b049d39f82d..fccf6a86cb141 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -846,6 +846,8 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) +- `ALLOWLISTED_DOMAINS`: ****: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKLISTED_DOMAINS`: ****: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. ## Mirror (`mirror`) @@ -902,11 +904,6 @@ MINIO_USE_SSL = false And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`. -## Migraions (`migration`) - -- `ALLOWLISTED_DOMAINS`: ****: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. -- `BLOCKLISTED_DOMAINS`: ****: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. - ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 7fff6622d5251..89bb7e084959b 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -313,6 +313,8 @@ IS_INPUT_FILE = false - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。 +- `ALLOWLISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLOCKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 ## LFS (`lfs`) @@ -363,11 +365,6 @@ MINIO_USE_SSL = false 然后你在 `[attachment]`, `[lfs]` 等中可以把这个名字用作 `STORAGE_TYPE` 的值。 -## Migraions (`migration`) - -- `ALLOWLISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 -- `BLOCKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 - ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 94bc84233265c..4e84f44df0c5c 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -37,7 +37,7 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { } if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { - if len(setting.Migration.AllowlistedDomains) > 0 { + if len(setting.Migrations.AllowlistedDomains) > 0 { if !allowlist.Match(u.Host) { return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) } @@ -345,12 +345,12 @@ var ( // Init migrations service func Init() error { var err error - allowlist, err = matchlist.NewMatchlist(setting.Migration.AllowlistedDomains...) + allowlist, err = matchlist.NewMatchlist(setting.Migrations.AllowlistedDomains...) if err != nil { return fmt.Errorf("init migration allowlist domains failed: %v", err) } - blocklist, err = matchlist.NewMatchlist(setting.Migration.BlocklistedDomains...) + blocklist, err = matchlist.NewMatchlist(setting.Migrations.BlocklistedDomains...) if err != nil { return fmt.Errorf("init migration blocklist domains failed: %v", err) } diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index 6cb547db5295a..df9989abe671c 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -13,7 +13,7 @@ import ( ) func TestMigrateWhiteBlocklist(t *testing.T) { - setting.Migration.AllowlistedDomains = []string{"github.com"} + setting.Migrations.AllowlistedDomains = []string{"github.com"} assert.NoError(t, Init()) allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") @@ -24,8 +24,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) { assert.True(t, allowed) assert.NoError(t, err) - setting.Migration.AllowlistedDomains = []string{} - setting.Migration.BlocklistedDomains = []string{"github.com"} + setting.Migrations.AllowlistedDomains = []string{} + setting.Migrations.BlocklistedDomains = []string{"github.com"} assert.NoError(t, Init()) allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go deleted file mode 100644 index 50bc98632fc25..0000000000000 --- a/modules/setting/migrate.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package setting - -import ( - "fmt" -) - -// Migration represents migrations' settings -var Migration = struct { - AllowlistedDomains []string - BlocklistedDomains []string -}{ - AllowlistedDomains: []string{}, - BlocklistedDomains: []string{}, -} - -// InitMigrationConfig represents load migration configurations -func InitMigrationConfig() error { - if err := Cfg.Section("migration").MapTo(&Migration); err != nil { - return fmt.Errorf("Failed to map Migration settings: %v", err) - } - return nil -} diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 51d6bbcf11be1..f23ad68e5a5d7 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -7,8 +7,10 @@ package setting var ( // Migrations settings Migrations = struct { - MaxAttempts int - RetryBackoff int + MaxAttempts int + RetryBackoff int + AllowlistedDomains []string + BlocklistedDomains []string }{ MaxAttempts: 3, RetryBackoff: 3, @@ -19,4 +21,7 @@ func newMigrationsService() { sec := Cfg.Section("migrations") Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) + + Migrations.AllowlistedDomains = sec.Key("ALLOWLISTED_DOMAINS").Strings(",") + Migrations.BlocklistedDomains = sec.Key("BLOCKLISTED_DOMAINS").Strings(",") } From 9f5e0de71c54b5fcc3ba387ed077dcab28bf4167 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 23 Nov 2020 23:04:42 +0100 Subject: [PATCH 12/23] convert domains lower case --- modules/setting/migrations.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index f23ad68e5a5d7..582d0f825fec0 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -4,6 +4,10 @@ package setting +import ( + "strings" +) + var ( // Migrations settings Migrations = struct { @@ -23,5 +27,11 @@ func newMigrationsService() { Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) Migrations.AllowlistedDomains = sec.Key("ALLOWLISTED_DOMAINS").Strings(",") + for i := range Migrations.AllowlistedDomains { + Migrations.AllowlistedDomains[i] = strings.ToLower(Migrations.AllowlistedDomains[i]) + } Migrations.BlocklistedDomains = sec.Key("BLOCKLISTED_DOMAINS").Strings(",") + for i := range Migrations.BlocklistedDomains { + Migrations.BlocklistedDomains[i] = strings.ToLower(Migrations.BlocklistedDomains[i]) + } } From 37f45f4c34e011daa38ce7db5d88f25970366d06 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 23 Nov 2020 23:31:22 +0100 Subject: [PATCH 13/23] dont store unused value --- modules/matchlist/matchlist.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go index 666f0d8e00973..b65ed909dc1c7 100644 --- a/modules/matchlist/matchlist.go +++ b/modules/matchlist/matchlist.go @@ -12,7 +12,6 @@ import ( // Matchlist represents a block or allow list type Matchlist struct { - rules []string ruleGlobs []glob.Glob } @@ -22,11 +21,10 @@ func NewMatchlist(rules ...string) (*Matchlist, error) { rules[i] = strings.ToLower(rules[i]) } list := Matchlist{ - rules: rules, ruleGlobs: make([]glob.Glob, 0, len(rules)), } - for _, rule := range list.rules { + for _, rule := range rules { rg, err := glob.Compile(rule) if err != nil { return nil, err From e0934b8fd5842f0072486a494d4261052e2e4d0b Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 00:34:44 +0100 Subject: [PATCH 14/23] Block private addresses for migration by default --- custom/conf/app.example.ini | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 5 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 5 +- modules/migrations/migrate.go | 50 ++++++++++++++----- modules/setting/migrations.go | 3 ++ 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 33c8144d054ed..2690320a68d90 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1193,6 +1193,8 @@ ALLOWLISTED_DOMAINS = ; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. ; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. BLOCKLISTED_DOMAINS = +; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 +ALLOW_LOCALNETWORKS = false ; default storage for attachments, lfs and avatars [storage] diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index fccf6a86cb141..f034bea08ae07 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -846,8 +846,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) -- `ALLOWLISTED_DOMAINS`: ****: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. -- `BLOCKLISTED_DOMAINS`: ****: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. +- `ALLOWLISTED_DOMAINS`: **\**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKLISTED_DOMAINS`: **\**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. +- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 ## Mirror (`mirror`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 89bb7e084959b..aeaa1cb433503 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -313,8 +313,9 @@ IS_INPUT_FILE = false - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。 -- `ALLOWLISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 -- `BLOCKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 +- `ALLOWLISTED_DOMAINS`: **\**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLOCKLISTED_DOMAINS`: **\**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 +- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 ## LFS (`lfs`) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 4e84f44df0c5c..b6ee54a90ea08 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -8,6 +8,7 @@ package migrations import ( "context" "fmt" + "net" "net/url" "strings" @@ -23,6 +24,10 @@ type MigrateOptions = base.MigrateOptions var ( factories []base.DownloaderFactory + + allowList *matchlist.Matchlist + blockList *matchlist.Matchlist + privateNetworkList *matchlist.Matchlist ) // RegisterDownloaderFactory registers a downloader factory @@ -38,12 +43,27 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { if len(setting.Migrations.AllowlistedDomains) > 0 { - if !allowlist.Match(u.Host) { - return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + if !allowList.Match(u.Host) { + return false, fmt.Errorf("migrate from '%v' is not allowed", u.Host) + } + } else { + if blockList.Match(u.Host) { + return false, fmt.Errorf("migrate from '%v' is not allowed", u.Host) } + } + } + + if !setting.Migrations.AllowLocalNetworks { + addrList, err := net.LookupIP(u.Host) + if err != nil { + return false, fmt.Errorf("migrate from '%v' failed: unknown hostname", u.Host) } else { - if blocklist.Match(u.Host) { - return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + for _, addr := range addrList { + // workaround with **privateNetworkList** for RFC 1918, as long as "net" do not support it + // https://github.com/golang/go/issues/29146 + if !addr.IsGlobalUnicast() || privateNetworkList.Match(addr.String()) { + return false, fmt.Errorf("migrate from '%v' not allowed, has private network address '%s'", u.Host, addr.String()) + } } } } @@ -337,22 +357,26 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return nil } -var ( - allowlist *matchlist.Matchlist - blocklist *matchlist.Matchlist -) - // Init migrations service func Init() error { var err error - allowlist, err = matchlist.NewMatchlist(setting.Migrations.AllowlistedDomains...) + allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowlistedDomains...) if err != nil { - return fmt.Errorf("init migration allowlist domains failed: %v", err) + return fmt.Errorf("init migration allowList domains failed: %v", err) } - blocklist, err = matchlist.NewMatchlist(setting.Migrations.BlocklistedDomains...) + blockList, err = matchlist.NewMatchlist(setting.Migrations.BlocklistedDomains...) if err != nil { - return fmt.Errorf("init migration blocklist domains failed: %v", err) + return fmt.Errorf("init migration blockList domains failed: %v", err) } + + // TODO: remove if https://github.com/golang/go/issues/29146 got resolved + privateNetworkList, _ = matchlist.NewMatchlist( + "localhost", // localhost + "{10,127}\\.[0-9]*\\.[0-9]*\\.[0-9]*", // 127.0.0.0/8 & 10.0.0.0/8 + "172\\.{1[6-9],2[0-9],3[01]}\\.[0-9]*\\.[0-9]*", // 172.16.0.0/12 + "192\\.168\\.[0-9]*\\.[0-9]*", // 192.168.0.0/16 + ) + return nil } diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 582d0f825fec0..9b4763c39a227 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -15,6 +15,7 @@ var ( RetryBackoff int AllowlistedDomains []string BlocklistedDomains []string + AllowLocalNetworks bool }{ MaxAttempts: 3, RetryBackoff: 3, @@ -34,4 +35,6 @@ func newMigrationsService() { for i := range Migrations.BlocklistedDomains { Migrations.BlocklistedDomains[i] = strings.ToLower(Migrations.BlocklistedDomains[i]) } + + Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) } From 43982b417bc1dbee66091e81a56f2cb964b5dca4 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 00:44:42 +0100 Subject: [PATCH 15/23] fix lint --- custom/conf/app.example.ini | 2 +- modules/migrations/migrate.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 2690320a68d90..6b3adf2b4de37 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1193,7 +1193,7 @@ ALLOWLISTED_DOMAINS = ; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. ; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. BLOCKLISTED_DOMAINS = -; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 +; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) ALLOW_LOCALNETWORKS = false ; default storage for attachments, lfs and avatars diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index b6ee54a90ea08..7a24fec7161a8 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -57,13 +57,13 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { addrList, err := net.LookupIP(u.Host) if err != nil { return false, fmt.Errorf("migrate from '%v' failed: unknown hostname", u.Host) - } else { - for _, addr := range addrList { - // workaround with **privateNetworkList** for RFC 1918, as long as "net" do not support it - // https://github.com/golang/go/issues/29146 - if !addr.IsGlobalUnicast() || privateNetworkList.Match(addr.String()) { - return false, fmt.Errorf("migrate from '%v' not allowed, has private network address '%s'", u.Host, addr.String()) - } + } + + for _, addr := range addrList { + // workaround with **privateNetworkList** for RFC 1918, as long as "net" do not support it + // https://github.com/golang/go/issues/29146 + if !addr.IsGlobalUnicast() || privateNetworkList.Match(addr.String()) { + return false, fmt.Errorf("migrate from '%v' not allowed, has private network address '%s'", u.Host, addr.String()) } } } From 6725fd5aabb6f4a9764632889a8c8da84d503664 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 02:29:27 +0100 Subject: [PATCH 16/23] use proposed-upstream func to detect private IP addr --- modules/migrations/migrate.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 7a24fec7161a8..b2d6a4d41e0d0 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -25,9 +25,8 @@ type MigrateOptions = base.MigrateOptions var ( factories []base.DownloaderFactory - allowList *matchlist.Matchlist - blockList *matchlist.Matchlist - privateNetworkList *matchlist.Matchlist + allowList *matchlist.Matchlist + blockList *matchlist.Matchlist ) // RegisterDownloaderFactory registers a downloader factory @@ -58,11 +57,8 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { if err != nil { return false, fmt.Errorf("migrate from '%v' failed: unknown hostname", u.Host) } - for _, addr := range addrList { - // workaround with **privateNetworkList** for RFC 1918, as long as "net" do not support it - // https://github.com/golang/go/issues/29146 - if !addr.IsGlobalUnicast() || privateNetworkList.Match(addr.String()) { + if isIPPrivate(addr) || !addr.IsGlobalUnicast() { return false, fmt.Errorf("migrate from '%v' not allowed, has private network address '%s'", u.Host, addr.String()) } } @@ -370,13 +366,18 @@ func Init() error { return fmt.Errorf("init migration blockList domains failed: %v", err) } - // TODO: remove if https://github.com/golang/go/issues/29146 got resolved - privateNetworkList, _ = matchlist.NewMatchlist( - "localhost", // localhost - "{10,127}\\.[0-9]*\\.[0-9]*\\.[0-9]*", // 127.0.0.0/8 & 10.0.0.0/8 - "172\\.{1[6-9],2[0-9],3[01]}\\.[0-9]*\\.[0-9]*", // 172.16.0.0/12 - "192\\.168\\.[0-9]*\\.[0-9]*", // 192.168.0.0/16 - ) - return nil } + +// isIPPrivate reports whether ip is a private address, according to +// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses). +// from https://github.com/golang/go/pull/42793 +// TODO remove if https://github.com/golang/go/issues/29146 got resolved +func isIPPrivate(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || + (ip4[0] == 172 && ip4[1]&0xf0 == 16) || + (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc +} From 6b8ecc4e73ede0f853b6a6aaee3a94fca70b5640 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 02:53:48 +0100 Subject: [PATCH 17/23] a nit --- modules/migrations/migrate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index b2d6a4d41e0d0..6067d05c07b46 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -53,13 +53,13 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { } if !setting.Migrations.AllowLocalNetworks { - addrList, err := net.LookupIP(u.Host) + addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) if err != nil { return false, fmt.Errorf("migrate from '%v' failed: unknown hostname", u.Host) } for _, addr := range addrList { if isIPPrivate(addr) || !addr.IsGlobalUnicast() { - return false, fmt.Errorf("migrate from '%v' not allowed, has private network address '%s'", u.Host, addr.String()) + return false, fmt.Errorf("migrate from '%v' not allowed, host resolve to private ip address '%s'", u.Host, addr.String()) } } } From 6ef72679fa79bdc2e13018b5b579b367f07593b7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 03:38:32 +0100 Subject: [PATCH 18/23] add own error for blocked migration, add tests, imprufe api --- integrations/api_repo_test.go | 11 ++++++++++- models/error.go | 27 +++++++++++++++++++++++++++ modules/migrations/migrate.go | 18 +++++++++--------- routers/api/v1/repo/migrate.go | 2 ++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/integrations/api_repo_test.go b/integrations/api_repo_test.go index da87db7b1b5ba..6d4b0848dac1a 100644 --- a/integrations/api_repo_test.go +++ b/integrations/api_repo_test.go @@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) { {ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, {ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, {ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, + {ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity}, + {ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity}, } defer prepareTestEnv(t)() @@ -325,8 +327,15 @@ func TestAPIRepoMigrate(t *testing.T) { if resp.Code == http.StatusUnprocessableEntity { respJSON := map[string]string{} DecodeJSON(t, resp, &respJSON) - if assert.Equal(t, respJSON["message"], "Remote visit addressed rate limitation.") { + switch respJSON["message"] { + case "Remote visit addressed rate limitation.": t.Log("test hit github rate limitation") + case "migrate from '10.0.0.1' not allowed, host resolve to private ip address '10.0.0.1'": + assert.EqualValues(t, "private-ip", testCase.repoName) + case "migrate from 'localhost:3000' not allowed, host resolve to private ip address '::1'": + assert.EqualValues(t, "local-ip", testCase.repoName) + default: + t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) } } else { assert.EqualValues(t, testCase.expectedStatus, resp.Code) diff --git a/models/error.go b/models/error.go index 83354ff173d55..0a87c05eb6801 100644 --- a/models/error.go +++ b/models/error.go @@ -1019,6 +1019,33 @@ func IsErrWontSign(err error) bool { return ok } +// ErrMigrateFromHost explains why a migration failed caused by source host +type ErrMigrateFromHost struct { + Host string + NotResolvedIP bool + BlockedDomain bool + PrivateNet string +} + +func (e *ErrMigrateFromHost) Error() string { + if e.BlockedDomain { + return fmt.Sprintf("migrate from '%s' is not allowed", e.Host) + } + if e.NotResolvedIP { + return fmt.Sprintf("migrate from '%s' failed: unknown hostname", e.Host) + } + if len(e.PrivateNet) != 0 { + return fmt.Sprintf("migrate from '%s' not allowed, host resolve to private ip address '%s'", e.Host, e.PrivateNet) + } + return fmt.Sprintf("migrate from '%s failed'", e.Host) +} + +// IsErrMigrateFromHost checks if an error is a ErrMigrateFromHost +func IsErrMigrateFromHost(err error) bool { + _, ok := err.(*ErrMigrateFromHost) + return ok +} + // __________ .__ // \______ \____________ ____ ____ | |__ // | | _/\_ __ \__ \ / \_/ ___\| | \ diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 6067d05c07b46..942c1e3e74185 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -34,20 +34,20 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { factories = append(factories, factory) } -func isMigrateURLAllowed(remoteURL string) (bool, error) { +func isMigrateURLAllowed(remoteURL string) error { u, err := url.Parse(strings.ToLower(remoteURL)) if err != nil { - return false, err + return err } if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { if len(setting.Migrations.AllowlistedDomains) > 0 { if !allowList.Match(u.Host) { - return false, fmt.Errorf("migrate from '%v' is not allowed", u.Host) + return &models.ErrMigrateFromHost{Host: u.Host, BlockedDomain: true} } } else { if blockList.Match(u.Host) { - return false, fmt.Errorf("migrate from '%v' is not allowed", u.Host) + return &models.ErrMigrateFromHost{Host: u.Host, BlockedDomain: true} } } } @@ -55,22 +55,22 @@ func isMigrateURLAllowed(remoteURL string) (bool, error) { if !setting.Migrations.AllowLocalNetworks { addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) if err != nil { - return false, fmt.Errorf("migrate from '%v' failed: unknown hostname", u.Host) + return &models.ErrMigrateFromHost{Host: u.Host, NotResolvedIP: true} } for _, addr := range addrList { if isIPPrivate(addr) || !addr.IsGlobalUnicast() { - return false, fmt.Errorf("migrate from '%v' not allowed, host resolve to private ip address '%s'", u.Host, addr.String()) + return &models.ErrMigrateFromHost{Host: u.Host, PrivateNet: addr.String()} } } } - return true, nil + return nil } // MigrateRepository migrate repository according MigrateOptions func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { - allowed, err := isMigrateURLAllowed(opts.CloneAddr) - if !allowed { + err := isMigrateURLAllowed(opts.CloneAddr) + if err != nil { return nil, err } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index f9cddbb7cdce8..6de02212b6c66 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -212,6 +212,8 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) case models.IsErrNamePatternNotAllowed(err): ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) + case models.IsErrMigrateFromHost(err): + ctx.Error(http.StatusUnprocessableEntity, "", err) default: err = util.URLSanitizedError(err, remoteAddr) if strings.Contains(err.Error(), "Authentication failed") || From c66cf83091c70e2b264226ca3c038c48101ce4ce Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 03:42:19 +0100 Subject: [PATCH 19/23] fix test --- modules/migrations/migrate_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index df9989abe671c..75905c28edcba 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -16,23 +16,19 @@ func TestMigrateWhiteBlocklist(t *testing.T) { setting.Migrations.AllowlistedDomains = []string{"github.com"} assert.NoError(t, Init()) - allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") - assert.False(t, allowed) + err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") assert.Error(t, err) - allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") - assert.True(t, allowed) + err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") assert.NoError(t, err) setting.Migrations.AllowlistedDomains = []string{} setting.Migrations.BlocklistedDomains = []string{"github.com"} assert.NoError(t, Init()) - allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") - assert.True(t, allowed) + err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") assert.NoError(t, err) - allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") - assert.False(t, allowed) + err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") assert.Error(t, err) } From 3b57ffc0416f6f76b2b6155febf70c9a1f104390 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 24 Nov 2020 04:04:50 +0100 Subject: [PATCH 20/23] fix-if-localhost-is-ipv4 --- integrations/api_repo_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/api_repo_test.go b/integrations/api_repo_test.go index 6d4b0848dac1a..3a6e197715567 100644 --- a/integrations/api_repo_test.go +++ b/integrations/api_repo_test.go @@ -332,7 +332,8 @@ func TestAPIRepoMigrate(t *testing.T) { t.Log("test hit github rate limitation") case "migrate from '10.0.0.1' not allowed, host resolve to private ip address '10.0.0.1'": assert.EqualValues(t, "private-ip", testCase.repoName) - case "migrate from 'localhost:3000' not allowed, host resolve to private ip address '::1'": + case "migrate from 'localhost:3000' not allowed, host resolve to private ip address '::1'", + "migrate from 'localhost:3000' not allowed, host resolve to private ip address '127.0.0.1'": assert.EqualValues(t, "local-ip", testCase.repoName) default: t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) From 8372dd1ce984ee10af94a8f0033f033e6baa45be Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 26 Nov 2020 21:31:51 +0100 Subject: [PATCH 21/23] rename error & error message --- integrations/api_repo_test.go | 6 +++--- models/error.go | 22 +++++++++------------- modules/migrations/migrate.go | 8 ++++---- routers/api/v1/repo/migrate.go | 2 +- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/integrations/api_repo_test.go b/integrations/api_repo_test.go index 3a6e197715567..8294a01773351 100644 --- a/integrations/api_repo_test.go +++ b/integrations/api_repo_test.go @@ -330,10 +330,10 @@ func TestAPIRepoMigrate(t *testing.T) { switch respJSON["message"] { case "Remote visit addressed rate limitation.": t.Log("test hit github rate limitation") - case "migrate from '10.0.0.1' not allowed, host resolve to private ip address '10.0.0.1'": + case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'": assert.EqualValues(t, "private-ip", testCase.repoName) - case "migrate from 'localhost:3000' not allowed, host resolve to private ip address '::1'", - "migrate from 'localhost:3000' not allowed, host resolve to private ip address '127.0.0.1'": + case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'", + "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'": assert.EqualValues(t, "local-ip", testCase.repoName) default: t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) diff --git a/models/error.go b/models/error.go index 0a87c05eb6801..7f1eda1b14eb5 100644 --- a/models/error.go +++ b/models/error.go @@ -1019,30 +1019,26 @@ func IsErrWontSign(err error) bool { return ok } -// ErrMigrateFromHost explains why a migration failed caused by source host -type ErrMigrateFromHost struct { +// ErrMigrationNotAllowed explains why a migration from an url is not allowed +type ErrMigrationNotAllowed struct { Host string NotResolvedIP bool - BlockedDomain bool PrivateNet string } -func (e *ErrMigrateFromHost) Error() string { - if e.BlockedDomain { - return fmt.Sprintf("migrate from '%s' is not allowed", e.Host) - } +func (e *ErrMigrationNotAllowed) Error() string { if e.NotResolvedIP { - return fmt.Sprintf("migrate from '%s' failed: unknown hostname", e.Host) + return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host) } if len(e.PrivateNet) != 0 { - return fmt.Sprintf("migrate from '%s' not allowed, host resolve to private ip address '%s'", e.Host, e.PrivateNet) + return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet) } - return fmt.Sprintf("migrate from '%s failed'", e.Host) + return fmt.Sprintf("migrate from '%s is not allowed'", e.Host) } -// IsErrMigrateFromHost checks if an error is a ErrMigrateFromHost -func IsErrMigrateFromHost(err error) bool { - _, ok := err.(*ErrMigrateFromHost) +// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed +func IsErrMigrationNotAllowed(err error) bool { + _, ok := err.(*ErrMigrationNotAllowed) return ok } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 942c1e3e74185..c6ea647fb9cea 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -43,11 +43,11 @@ func isMigrateURLAllowed(remoteURL string) error { if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { if len(setting.Migrations.AllowlistedDomains) > 0 { if !allowList.Match(u.Host) { - return &models.ErrMigrateFromHost{Host: u.Host, BlockedDomain: true} + return &models.ErrMigrationNotAllowed{Host: u.Host} } } else { if blockList.Match(u.Host) { - return &models.ErrMigrateFromHost{Host: u.Host, BlockedDomain: true} + return &models.ErrMigrationNotAllowed{Host: u.Host} } } } @@ -55,11 +55,11 @@ func isMigrateURLAllowed(remoteURL string) error { if !setting.Migrations.AllowLocalNetworks { addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) if err != nil { - return &models.ErrMigrateFromHost{Host: u.Host, NotResolvedIP: true} + return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true} } for _, addr := range addrList { if isIPPrivate(addr) || !addr.IsGlobalUnicast() { - return &models.ErrMigrateFromHost{Host: u.Host, PrivateNet: addr.String()} + return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()} } } } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 6de02212b6c66..68ab7e4897e67 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -212,7 +212,7 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) case models.IsErrNamePatternNotAllowed(err): ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) - case models.IsErrMigrateFromHost(err): + case models.IsErrMigrationNotAllowed(err): ctx.Error(http.StatusUnprocessableEntity, "", err) default: err = util.URLSanitizedError(err, remoteAddr) From bb5ce58fbce3eb24f8794956e2abe99d21b1738d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 28 Nov 2020 03:43:37 +0100 Subject: [PATCH 22/23] rename setting options --- .../doc/advanced/config-cheat-sheet.en-us.md | 4 ++-- .../doc/advanced/config-cheat-sheet.zh-cn.md | 4 ++-- modules/migrations/migrate.go | 6 +++--- modules/migrations/migrate_test.go | 6 +++--- modules/setting/migrations.go | 16 ++++++++-------- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index f034bea08ae07..5d029fd31ac06 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -846,8 +846,8 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) -- `ALLOWLISTED_DOMAINS`: **\**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. -- `BLOCKLISTED_DOMAINS`: **\**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWLISTED_DOMAINS` is not blank, this option will be ignored. +- `ALLOWED_DOMAINS`: **\**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLOCKED_DOMAINS`: **\**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 ## Mirror (`mirror`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index aeaa1cb433503..597773a0ae980 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -313,8 +313,8 @@ IS_INPUT_FILE = false - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间，单位秒。 -- `ALLOWLISTED_DOMAINS`: **\**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 -- `BLOCKLISTED_DOMAINS`: **\**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWLISTED_DOMAINS` 不为空，此选项将会被忽略。 +- `ALLOWED_DOMAINS`: **\**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLOCKED_DOMAINS`: **\**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 ## LFS (`lfs`) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index c6ea647fb9cea..b3ecb8114a402 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -41,7 +41,7 @@ func isMigrateURLAllowed(remoteURL string) error { } if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { - if len(setting.Migrations.AllowlistedDomains) > 0 { + if len(setting.Migrations.AllowedDomains) > 0 { if !allowList.Match(u.Host) { return &models.ErrMigrationNotAllowed{Host: u.Host} } @@ -356,12 +356,12 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // Init migrations service func Init() error { var err error - allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowlistedDomains...) + allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) if err != nil { return fmt.Errorf("init migration allowList domains failed: %v", err) } - blockList, err = matchlist.NewMatchlist(setting.Migrations.BlocklistedDomains...) + blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) if err != nil { return fmt.Errorf("init migration blockList domains failed: %v", err) } diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index 75905c28edcba..3bad5cfd736be 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -13,7 +13,7 @@ import ( ) func TestMigrateWhiteBlocklist(t *testing.T) { - setting.Migrations.AllowlistedDomains = []string{"github.com"} + setting.Migrations.AllowedDomains = []string{"github.com"} assert.NoError(t, Init()) err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") @@ -22,8 +22,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) { err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") assert.NoError(t, err) - setting.Migrations.AllowlistedDomains = []string{} - setting.Migrations.BlocklistedDomains = []string{"github.com"} + setting.Migrations.AllowedDomains = []string{} + setting.Migrations.BlockedDomains = []string{"github.com"} assert.NoError(t, Init()) err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 9b4763c39a227..7808df5280756 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -13,8 +13,8 @@ var ( Migrations = struct { MaxAttempts int RetryBackoff int - AllowlistedDomains []string - BlocklistedDomains []string + AllowedDomains []string + BlockedDomains []string AllowLocalNetworks bool }{ MaxAttempts: 3, @@ -27,13 +27,13 @@ func newMigrationsService() { Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) - Migrations.AllowlistedDomains = sec.Key("ALLOWLISTED_DOMAINS").Strings(",") - for i := range Migrations.AllowlistedDomains { - Migrations.AllowlistedDomains[i] = strings.ToLower(Migrations.AllowlistedDomains[i]) + Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") + for i := range Migrations.AllowedDomains { + Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) } - Migrations.BlocklistedDomains = sec.Key("BLOCKLISTED_DOMAINS").Strings(",") - for i := range Migrations.BlocklistedDomains { - Migrations.BlocklistedDomains[i] = strings.ToLower(Migrations.BlocklistedDomains[i]) + Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") + for i := range Migrations.BlockedDomains { + Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) } Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) From e13cd150499aebdecc2e6e5bc0cd3d03c1f55e17 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 28 Nov 2020 14:36:43 +0000 Subject: [PATCH 23/23] Apply suggestions from code review --- custom/conf/app.example.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 6b3adf2b4de37..8c5e6b05a90f6 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1187,12 +1187,12 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" MAX_ATTEMPTS = 3 ; Backoff time per http/https request retry (seconds) RETRY_BACKOFF = 3 -; Allowlist for migrating, default is blank. Blank means everything will be allowed. +; Allowed domains for migrating, default is blank. Blank means everything will be allowed. ; Multiple domains could be separated by commas. -ALLOWLISTED_DOMAINS = +ALLOWED_DOMAINS = ; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. -; When ALLOWLISTED_DOMAINS is not blank, this option will be ignored. -BLOCKLISTED_DOMAINS = +; When ALLOWED_DOMAINS is not blank, this option will be ignored. +BLOCKED_DOMAINS = ; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) ALLOW_LOCALNETWORKS = false