From 20674dd05da909b42cbdd07a6682fdf1d980f011 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 9 Nov 2022 07:34:27 +0100 Subject: [PATCH] Add package registry quota limits (#21584) Related #20471 This PR adds global quota limits for the package registry. Settings for individual users/orgs can be added in a seperate PR using the settings table. Co-authored-by: Lauris BH Co-authored-by: Lunny Xiao --- cmd/migrate_storage_test.go | 5 +- custom/conf/app.example.ini | 29 ++++++ .../doc/advanced/config-cheat-sheet.en-us.md | 14 +++ models/packages/package_file.go | 10 ++ models/packages/package_version.go | 9 ++ modules/setting/packages.go | 50 +++++++++- modules/setting/packages_test.go | 31 ++++++ routers/api/packages/composer/composer.go | 14 ++- routers/api/packages/conan/conan.go | 14 ++- routers/api/packages/generic/generic.go | 14 ++- routers/api/packages/helm/helm.go | 10 +- routers/api/packages/maven/maven.go | 10 +- routers/api/packages/npm/npm.go | 14 ++- routers/api/packages/nuget/nuget.go | 28 ++++-- routers/api/packages/pub/pub.go | 14 ++- routers/api/packages/pypi/pypi.go | 14 ++- routers/api/packages/rubygems/rubygems.go | 14 ++- routers/api/packages/vagrant/vagrant.go | 14 ++- services/packages/packages.go | 97 ++++++++++++++++++- tests/integration/api_packages_test.go | 34 +++++++ 20 files changed, 378 insertions(+), 61 deletions(-) create mode 100644 modules/setting/packages_test.go diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 0d264ef5a1bdc..7051591ad68e0 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -44,8 +44,9 @@ func TestMigratePackages(t *testing.T) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: "a.go", }, - Data: buf, - IsLead: true, + Creator: creator, + Data: buf, + IsLead: true, }) assert.NoError(t, err) assert.NotNil(t, v) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b59ceee4f1db9..b46dfc20a9696 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2335,6 +2335,35 @@ ROUTER = console ;; ;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` ;CHUNKED_UPLOAD_PATH = tmp/package-upload +;; +;; Maxmimum count of package versions a single owner can have (`-1` means no limits) +;LIMIT_TOTAL_OWNER_COUNT = -1 +;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_TOTAL_OWNER_SIZE = -1 +;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_COMPOSER = -1 +;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CONAN = -1 +;; Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CONTAINER = -1 +;; Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_GENERIC = -1 +;; Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_HELM = -1 +;; Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_MAVEN = -1 +;; Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_NPM = -1 +;; Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_NUGET = -1 +;; Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_PUB = -1 +;; Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_PYPI = -1 +;; Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_RUBYGEMS = -1 +;; Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_VAGRANT = -1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index df1911934c885..28bcaf29afdcd 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1138,6 +1138,20 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `ENABLED`: **true**: Enable/Disable package registry capabilities - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` +- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits) +- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_GENERIC`: **-1**: Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_HELM`: **-1**: Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_MAVEN`: **-1**: Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_NPM`: **-1**: Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_NUGET`: **-1**: Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_PUB`: **-1**: Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_PYPI`: **-1**: Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_RUBYGEMS`: **-1**: Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_VAGRANT`: **-1**: Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ## Mirror (`mirror`) diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 8f304ce8ac42a..9f6284af0763d 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -199,3 +199,13 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag count, err := sess.FindAndCount(&pfs) return pfs, count, err } + +// CalculateBlobSize sums up all blob sizes matching the search options. +// It does NOT respect the deduplication of blobs. +func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) { + return db.GetEngine(ctx). + Table("package_file"). + Where(opts.toConds()). + Join("INNER", "package_blob", "package_blob.id = package_file.blob_id"). + SumInt(new(PackageBlob), "size") +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 782261c575dcf..48c6aa7d607fa 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P count, err := sess.FindAndCount(&pvs) return pvs, count, err } + +// CountVersions counts all versions of packages matching the search options +func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) { + return db.GetEngine(ctx). + Where(opts.toConds()). + Table("package_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Count(new(PackageVersion)) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 5e0f2a3b03da9..62201032c7403 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -5,11 +5,15 @@ package setting import ( + "math" "net/url" "os" "path/filepath" "code.gitea.io/gitea/modules/log" + + "github.com/dustin/go-humanize" + ini "gopkg.in/ini.v1" ) // Package registry settings @@ -19,8 +23,24 @@ var ( Enabled bool ChunkedUploadPath string RegistryHost string + + LimitTotalOwnerCount int64 + LimitTotalOwnerSize int64 + LimitSizeComposer int64 + LimitSizeConan int64 + LimitSizeContainer int64 + LimitSizeGeneric int64 + LimitSizeHelm int64 + LimitSizeMaven int64 + LimitSizeNpm int64 + LimitSizeNuGet int64 + LimitSizePub int64 + LimitSizePyPI int64 + LimitSizeRubyGems int64 + LimitSizeVagrant int64 }{ - Enabled: true, + Enabled: true, + LimitTotalOwnerCount: -1, } ) @@ -43,4 +63,32 @@ func newPackages() { if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) } + + Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") + Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") + Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") + Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") + Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") + Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") + Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN") + Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM") + Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET") + Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB") + Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI") + Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") + Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") +} + +func mustBytes(section *ini.Section, key string) int64 { + const noLimit = "-1" + + value := section.Key(key).MustString(noLimit) + if value == noLimit { + return -1 + } + bytes, err := humanize.ParseBytes(value) + if err != nil || bytes > math.MaxInt64 { + return -1 + } + return int64(bytes) } diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go new file mode 100644 index 0000000000000..059273dce4be1 --- /dev/null +++ b/modules/setting/packages_test.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" + ini "gopkg.in/ini.v1" +) + +func TestMustBytes(t *testing.T) { + test := func(value string) int64 { + sec, _ := ini.Empty().NewSection("test") + sec.NewKey("VALUE", value) + + return mustBytes(sec, "VALUE") + } + + assert.EqualValues(t, -1, test("")) + assert.EqualValues(t, -1, test("-1")) + assert.EqualValues(t, 0, test("0")) + assert.EqualValues(t, 1, test("1")) + assert.EqualValues(t, 10000, test("10000")) + assert.EqualValues(t, 1000000, test("1 mb")) + assert.EqualValues(t, 1048576, test("1mib")) + assert.EqualValues(t, 1782579, test("1.7mib")) + assert.EqualValues(t, -1, test("1 yib")) // too large +} diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index 86ef7cbd9af58..92e83dbe79d2d 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -235,16 +235,20 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index dd078d6ad34d1..c8c9dc3e384bd 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -348,8 +348,9 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey Filename: strings.ToLower(filename), CompositeKey: fileKey, }, - Data: buf, - IsLead: isConanfileFile, + Creator: ctx.Doer, + Data: buf, + IsLead: isConanfileFile, Properties: map[string]string{ conan_module.PropertyRecipeUser: rref.User, conan_module.PropertyRecipeChannel: rref.Channel, @@ -416,11 +417,14 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey pfci, ) if err != nil { - if err == packages_model.ErrDuplicatePackageFile { + switch err { + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 81891bec2646b..1bccc6764cf12 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -104,16 +104,20 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: filename, }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageFile { + switch err { + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusConflict, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index 9c85e0874fd46..662d9a5dda28f 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -186,17 +186,21 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: createFilename(metadata), }, + Creator: ctx.Doer, Data: buf, IsLead: true, OverwriteExisting: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusConflict, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index bf00c199f5639..de274b2046093 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -266,6 +266,7 @@ func UploadPackageFile(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: params.Filename, }, + Creator: ctx.Doer, Data: buf, IsLead: false, OverwriteExisting: params.IsMeta, @@ -312,11 +313,14 @@ func UploadPackageFile(ctx *context.Context) { pfci, ) if err != nil { - if err == packages_model.ErrDuplicatePackageFile { + switch err { + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 82dae0cf435d6..6d589bde3a541 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -180,16 +180,20 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: npmPackage.Filename, }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index e84aef3160f02..442d94243ba37 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -374,16 +374,20 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)), }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusConflict, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } @@ -428,8 +432,9 @@ func UploadSymbolPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)), }, - Data: buf, - IsLead: false, + Creator: ctx.Doer, + Data: buf, + IsLead: false, }, ) if err != nil { @@ -438,6 +443,8 @@ func UploadSymbolPackage(ctx *context.Context) { apiError(ctx, http.StatusNotFound, err) case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) default: apiError(ctx, http.StatusInternalServerError, err) } @@ -452,8 +459,9 @@ func UploadSymbolPackage(ctx *context.Context) { Filename: strings.ToLower(pdb.Name), CompositeKey: strings.ToLower(pdb.ID), }, - Data: pdb.Content, - IsLead: false, + Creator: ctx.Doer, + Data: pdb.Content, + IsLead: false, Properties: map[string]string{ nuget_module.PropertySymbolID: strings.ToLower(pdb.ID), }, @@ -463,6 +471,8 @@ func UploadSymbolPackage(ctx *context.Context) { switch err { case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) default: apiError(ctx, http.StatusInternalServerError, err) } diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index 9af0ceeb0e6b9..635147b6d0c44 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -199,16 +199,20 @@ func UploadPackageFile(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(pck.Version + ".tar.gz"), }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 4c8041c30cc4c..4853e6658bdc5 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -162,16 +162,20 @@ func UploadPackageFile(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: fileHeader.Filename, }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageFile { + switch err { + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index 319c94b91fee9..eeae21146cbe8 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -242,16 +242,20 @@ func UploadPackageFile(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: filename, }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageVersion { + switch err { + case packages_model.ErrDuplicatePackageVersion: apiError(ctx, http.StatusBadRequest, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 7750e5dc4b2ba..31ac56a532c75 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -193,19 +193,23 @@ func UploadPackageFile(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(boxProvider), }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, Properties: map[string]string{ vagrant_module.PropertyProvider: strings.TrimSuffix(boxProvider, ".box"), }, }, ) if err != nil { - if err == packages_model.ErrDuplicatePackageFile { + switch err { + case packages_model.ErrDuplicatePackageFile: apiError(ctx, http.StatusConflict, err) - return + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) } - apiError(ctx, http.StatusInternalServerError, err) return } diff --git a/services/packages/packages.go b/services/packages/packages.go index 96132eac0980d..443976e174b75 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -6,6 +6,7 @@ package packages import ( "context" + "errors" "fmt" "io" "strings" @@ -19,10 +20,17 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" container_service "code.gitea.io/gitea/services/packages/container" ) +var ( + ErrQuotaTypeSize = errors.New("maximum allowed package type size exceeded") + ErrQuotaTotalSize = errors.New("maximum allowed package storage quota exceeded") + ErrQuotaTotalCount = errors.New("maximum allowed package count exceeded") +) + // PackageInfo describes a package type PackageInfo struct { Owner *user_model.User @@ -50,6 +58,7 @@ type PackageFileInfo struct { // PackageFileCreationInfo describes a package file to create type PackageFileCreationInfo struct { PackageFileInfo + Creator *user_model.User Data packages_module.HashedSizeReader IsLead bool Properties map[string]string @@ -78,7 +87,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreatio return nil, nil, err } - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, &pvci.PackageInfo, pfci) removeBlob := false defer func() { if blobCreated && removeBlob { @@ -164,6 +173,10 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all } if versionCreated { + if err := checkCountQuotaExceeded(ctx, pvci.Creator, pvci.Owner); err != nil { + return nil, false, err + } + for name, value := range pvci.VersionProperties { if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil { log.Error("Error setting package version property: %v", err) @@ -188,7 +201,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) ( return nil, nil, err } - pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci) + pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci) removeBlob := false defer func() { if removeBlob { @@ -224,9 +237,13 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag } } -func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { +func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) + if err := checkSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil { + return nil, nil, false, err + } + pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) if err != nil { log.Error("Error inserting package blob: %v", err) @@ -285,6 +302,80 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers return pf, pb, !exists, nil } +func checkCountQuotaExceeded(ctx context.Context, doer, owner *user_model.User) error { + if doer.IsAdmin { + return nil + } + + if setting.Packages.LimitTotalOwnerCount > -1 { + totalCount, err := packages_model.CountVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: owner.ID, + IsInternal: util.OptionalBoolFalse, + }) + if err != nil { + log.Error("CountVersions failed: %v", err) + return err + } + if totalCount > setting.Packages.LimitTotalOwnerCount { + return ErrQuotaTotalCount + } + } + + return nil +} + +func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, packageType packages_model.Type, uploadSize int64) error { + if doer.IsAdmin { + return nil + } + + var typeSpecificSize int64 + switch packageType { + case packages_model.TypeComposer: + typeSpecificSize = setting.Packages.LimitSizeComposer + case packages_model.TypeConan: + typeSpecificSize = setting.Packages.LimitSizeConan + case packages_model.TypeContainer: + typeSpecificSize = setting.Packages.LimitSizeContainer + case packages_model.TypeGeneric: + typeSpecificSize = setting.Packages.LimitSizeGeneric + case packages_model.TypeHelm: + typeSpecificSize = setting.Packages.LimitSizeHelm + case packages_model.TypeMaven: + typeSpecificSize = setting.Packages.LimitSizeMaven + case packages_model.TypeNpm: + typeSpecificSize = setting.Packages.LimitSizeNpm + case packages_model.TypeNuGet: + typeSpecificSize = setting.Packages.LimitSizeNuGet + case packages_model.TypePub: + typeSpecificSize = setting.Packages.LimitSizePub + case packages_model.TypePyPI: + typeSpecificSize = setting.Packages.LimitSizePyPI + case packages_model.TypeRubyGems: + typeSpecificSize = setting.Packages.LimitSizeRubyGems + case packages_model.TypeVagrant: + typeSpecificSize = setting.Packages.LimitSizeVagrant + } + if typeSpecificSize > -1 && typeSpecificSize < uploadSize { + return ErrQuotaTypeSize + } + + if setting.Packages.LimitTotalOwnerSize > -1 { + totalSize, err := packages_model.CalculateBlobSize(ctx, &packages_model.PackageFileSearchOptions{ + OwnerID: owner.ID, + }) + if err != nil { + log.Error("CalculateBlobSize failed: %v", err) + return err + } + if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize { + return ErrQuotaTotalSize + } + } + + return nil +} + // RemovePackageVersionByNameAndVersion deletes a package version and all associated files func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error { pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 25f5b3f2a12da..815685ea79960 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -16,6 +16,7 @@ import ( container_model "code.gitea.io/gitea/models/packages/container" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" packages_service "code.gitea.io/gitea/services/packages" "code.gitea.io/gitea/tests" @@ -166,6 +167,39 @@ func TestPackageAccess(t *testing.T) { uploadPackage(admin, user, http.StatusCreated) } +func TestPackageQuota(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + limitTotalOwnerCount, limitTotalOwnerSize, limitSizeGeneric := setting.Packages.LimitTotalOwnerCount, setting.Packages.LimitTotalOwnerSize, setting.Packages.LimitSizeGeneric + + admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10}) + + uploadPackage := func(doer *user_model.User, version string, expectedStatus int) { + url := fmt.Sprintf("/api/packages/%s/generic/test-package/%s/file.bin", user.Name, version) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1})) + AddBasicAuthHeader(req, doer.Name) + MakeRequest(t, req, expectedStatus) + } + + // Exceeded quota result in StatusForbidden for normal users but admins are always allowed to upload. + + setting.Packages.LimitTotalOwnerCount = 0 + uploadPackage(user, "1.0", http.StatusForbidden) + uploadPackage(admin, "1.0", http.StatusCreated) + setting.Packages.LimitTotalOwnerCount = limitTotalOwnerCount + + setting.Packages.LimitTotalOwnerSize = 0 + uploadPackage(user, "1.1", http.StatusForbidden) + uploadPackage(admin, "1.1", http.StatusCreated) + setting.Packages.LimitTotalOwnerSize = limitTotalOwnerSize + + setting.Packages.LimitSizeGeneric = 0 + uploadPackage(user, "1.2", http.StatusForbidden) + uploadPackage(admin, "1.2", http.StatusCreated) + setting.Packages.LimitSizeGeneric = limitSizeGeneric +} + func TestPackageCleanup(t *testing.T) { defer tests.PrepareTestEnv(t)()