From 1c160956daaba7ee5a00957790ec2e17a5a2933b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 31 Oct 2022 20:04:07 +0000 Subject: [PATCH 01/10] Add package cleanup rules. --- models/migrations/migrations.go | 2 + models/migrations/v232.go | 29 +++ models/packages/package.go | 15 ++ models/packages/package_cleanup_rule.go | 111 +++++++++ options/locale/locale_en-US.ini | 23 ++ routers/web/org/setting_packages.go | 87 +++++++ routers/web/shared/packages/packages.go | 232 ++++++++++++++++++ routers/web/user/setting/packages.go | 80 ++++++ routers/web/web.go | 29 +++ services/forms/package_form.go | 31 +++ services/packages/container/cleanup.go | 4 + services/packages/packages.go | 64 ++++- templates/org/settings/navbar.tmpl | 3 + templates/org/settings/packages.tmpl | 14 ++ .../settings/packages_cleanup_rules_edit.tmpl | 14 ++ .../packages_cleanup_rules_preview.tmpl | 13 + .../package/shared/cleanup_rules/edit.tmpl | 73 ++++++ .../package/shared/cleanup_rules/list.tmpl | 34 +++ .../package/shared/cleanup_rules/preview.tmpl | 34 +++ templates/user/settings/navbar.tmpl | 3 + templates/user/settings/packages.tmpl | 9 + .../settings/packages_cleanup_rules_edit.tmpl | 9 + .../packages_cleanup_rules_preview.tmpl | 8 + 23 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 models/migrations/v232.go create mode 100644 models/packages/package_cleanup_rule.go create mode 100644 routers/web/org/setting_packages.go create mode 100644 routers/web/shared/packages/packages.go create mode 100644 routers/web/user/setting/packages.go create mode 100644 services/forms/package_form.go create mode 100644 templates/org/settings/packages.tmpl create mode 100644 templates/org/settings/packages_cleanup_rules_edit.tmpl create mode 100644 templates/org/settings/packages_cleanup_rules_preview.tmpl create mode 100644 templates/package/shared/cleanup_rules/edit.tmpl create mode 100644 templates/package/shared/cleanup_rules/list.tmpl create mode 100644 templates/package/shared/cleanup_rules/preview.tmpl create mode 100644 templates/user/settings/packages.tmpl create mode 100644 templates/user/settings/packages_cleanup_rules_edit.tmpl create mode 100644 templates/user/settings/packages_cleanup_rules_preview.tmpl diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 31b88a7981103..07fad1845202d 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -425,6 +425,8 @@ var migrations = []Migration{ NewMigration("Add ConfidentialClient column (default true) to OAuth2Application table", addConfidentialClientColumnToOAuth2ApplicationTable), // v231 -> v232 NewMigration("Add index for hook_task", addIndexForHookTask), + // v232 -> v233 + NewMigration("Add package cleanup rule table", createPackageCleanupRuleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v232.go b/models/migrations/v232.go new file mode 100644 index 0000000000000..894703c43e4f3 --- /dev/null +++ b/models/migrations/v232.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func createPackageCleanupRuleTable(x *xorm.Engine) error { + type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(PackageCleanupRule)) +} diff --git a/models/packages/package.go b/models/packages/package.go index e39a7c4e411d4..cea04a0957955 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -45,6 +45,21 @@ const ( TypeVagrant Type = "vagrant" ) +var TypeList = []Type{ + TypeComposer, + TypeConan, + TypeContainer, + TypeGeneric, + TypeHelm, + TypeMaven, + TypeNpm, + TypeNuGet, + TypePub, + TypePyPI, + TypeRubyGems, + TypeVagrant, +} + // Name gets the name of the package type func (pt Type) Name() string { switch pt { diff --git a/models/packages/package_cleanup_rule.go b/models/packages/package_cleanup_rule.go new file mode 100644 index 0000000000000..dbed7545b84e7 --- /dev/null +++ b/models/packages/package_cleanup_rule.go @@ -0,0 +1,111 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "fmt" + "regexp" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist") + +func init() { + db.RegisterModel(new(PackageCleanupRule)) +} + +// PackageCleanupRule represents a rule which describes when to clean up package versions +type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + KeepPatternMatcher *regexp.Regexp `xorm:"-"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + RemovePatternMatcher *regexp.Regexp `xorm:"-"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` +} + +func (pcr *PackageCleanupRule) CompiledPattern() error { + if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil { + return nil + } + + if pcr.KeepPattern != "" { + var err error + pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern)) + if err != nil { + return err + } + } + + if pcr.RemovePattern != "" { + var err error + pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern)) + if err != nil { + return err + } + } + + return nil +} + +func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) { + _, err := db.GetEngine(ctx).Insert(pcr) + return pcr, err +} + +func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) { + pcr := &PackageCleanupRule{} + + has, err := db.GetEngine(ctx).ID(id).Get(pcr) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageCleanupRuleNotExist + } + return pcr, nil +} + +func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error { + _, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr) + return err +} + +func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) { + pcrs := make([]*PackageCleanupRule, 0, 10) + return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs) +} + +func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error { + _, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{}) + return err +} + +func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) { + return db.GetEngine(ctx). + Where("owner_id = ? AND type = ?", ownerID, packageType). + Exist(&PackageCleanupRule{}) +} + +func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error { + return db.Iterate( + ctx, + builder.Eq{"enabled": true}, + callback, + ) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1566dfc97d422..58b073ca3c68a 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -86,6 +86,9 @@ remove = Remove remove_all = Remove All edit = Edit +enabled = Enabled +disabled = Disabled + copy = Copy copy_url = Copy URL copy_branch = Copy branch name @@ -3177,3 +3180,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cleanuprules.title = Manage Cleanup Rules +owner.settings.cleanuprules.add = Add Cleanup Rule +owner.settings.cleanuprules.edit = Edit Cleanup Rule +owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more. +owner.settings.cleanuprules.preview = Cleanup Rule Preview +owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed. +owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages. +owner.settings.cleanuprules.enabled = Enabled +owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name +owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below. +owner.settings.cleanuprules.keep.count = Keep the most recent +owner.settings.cleanuprules.keep.count.1 = 1 version per package +owner.settings.cleanuprules.keep.count.n = %d versions per package +owner.settings.cleanuprules.keep.pattern = Keep versions matching +owner.settings.cleanuprules.keep.pattern.container = The latest version is always kept for Container packages. +owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them. +owner.settings.cleanuprules.remove.days = Remove versions older than +owner.settings.cleanuprules.remove.pattern = Remove versions matching +owner.settings.cleanuprules.success.update = Cleanup rule has been updated. +owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted. diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go new file mode 100644 index 0000000000000..c7edf4a18531c --- /dev/null +++ b/routers/web/org/setting_packages.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package org + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "org/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "org/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go new file mode 100644 index 0000000000000..82cb4d37c6a7d --- /dev/null +++ b/routers/web/shared/packages/packages.go @@ -0,0 +1,232 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "net/http" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/forms" + container_service "code.gitea.io/gitea/services/packages/container" +) + +const ( + tplSettingsPackages base.TplName = "user/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" +) + +func SetPackagesContext(ctx *context.Context, owner *user_model.User) { + pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID) + if err != nil { + ctx.ServerError("GetCleanupRulesByOwner", err) + return + } + + ctx.Data["CleanupRules"] = pcrs +} + +func SetRuleAddContext(ctx *context.Context) { + setRuleEditContext(ctx, nil) +} + +func SetRuleEditContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + setRuleEditContext(ctx, pcr) +} + +func setRuleEditContext(ctx *context.Context, pcr *packages_model.PackageCleanupRule) { + ctx.Data["IsEditRule"] = pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList +} + +func PerformRuleAddPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + performRuleEditPost(ctx, owner, nil, redirectURL, template) +} + +func PerformRuleEditPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + if form.Action == "remove" { + if err := packages_model.DeleteCleanupRuleByID(ctx, pcr.ID); err != nil { + ctx.ServerError("DeleteCleanupRuleByID", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.delete")) + ctx.Redirect(redirectURL) + } else { + performRuleEditPost(ctx, owner, pcr, redirectURL, template) + } +} + +func performRuleEditPost(ctx *context.Context, owner *user_model.User, pcr *packages_model.PackageCleanupRule, redirectURL string, template base.TplName) { + isEditRule := pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + pcr.Enabled = form.Enabled + pcr.OwnerID = owner.ID + pcr.KeepCount = form.KeepCount + pcr.KeepPattern = form.KeepPattern + pcr.RemoveDays = form.RemoveDays + pcr.RemovePattern = form.RemovePattern + pcr.MatchFullName = form.MatchFullName + + ctx.Data["IsEditRule"] = isEditRule + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList + + if ctx.HasError() { + ctx.HTML(http.StatusOK, template) + return + } + + if isEditRule { + if err := packages_model.UpdateCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("UpdateCleanupRule", err) + return + } + } else { + pcr.Type = packages_model.Type(form.Type) + + if has, err := packages_model.HasOwnerCleanupRuleForPackageType(ctx, owner.ID, pcr.Type); err != nil { + ctx.ServerError("HasOwnerCleanupRuleForPackageType", err) + return + } else if has { + ctx.Data["Err_Type"] = true + ctx.HTML(http.StatusOK, template) + return + } + + var err error + if pcr, err = packages_model.InsertCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("InsertCleanupRule", err) + return + } + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.update")) + ctx.Redirect(fmt.Sprintf("%s/rules/%d", redirectURL, pcr.ID)) +} + +func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + if err := pcr.CompiledPattern(); err != nil { + ctx.ServerError("CompiledPattern", err) + return + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + ctx.ServerError("GetPackagesByType", err) + return + } + + versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10) + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + ctx.ServerError("SearchVersions", err) + return + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + ctx.ServerError("ShouldBeSkipped", err) + return + } else if skip { + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + continue + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + ctx.ServerError("GetPackageDescriptor", err) + return + } + versionsToRemove = append(versionsToRemove, pd) + } + } + + ctx.Data["CleanupRule"] = pcr + ctx.Data["VersionsToRemove"] = versionsToRemove +} + +func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *packages_model.PackageCleanupRule { + id := ctx.FormInt64("id") + if id == 0 { + id = ctx.ParamsInt64("id") + } + + pcr, err := packages_model.GetCleanupRuleByID(ctx, id) + if err != nil { + if err == packages_model.ErrPackageCleanupRuleNotExist { + ctx.NotFound("", err) + } else { + ctx.ServerError("GetCleanupRuleByID", err) + } + return nil + } + + if pcr != nil && pcr.OwnerID == owner.ID { + return pcr + } + + ctx.NotFound("", fmt.Errorf("PackageCleanupRule[%v] not associated to owner %v", id, owner)) + + return nil +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go new file mode 100644 index 0000000000000..d44e904556d2b --- /dev/null +++ b/routers/web/user/setting/packages.go @@ -0,0 +1,80 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "user/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("settings") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/web.go b/routers/web/web.go index 0b16e756e1824..4aff8354982c0 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -443,6 +443,20 @@ func RegisterRoutes(m *web.Route) { m.Combo("/keys").Get(user_setting.Keys). Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost) m.Post("/keys/delete", user_setting.DeleteKey) + m.Group("/packages", func() { + m.Get("", user_setting.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", user_setting.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", user_setting.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleEditPost) + m.Get("/preview", user_setting.PackagesRulePreview) + }) + }) + }) m.Get("/organization", user_setting.Organization) m.Get("/repos", user_setting.Repos) m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository) @@ -751,6 +765,21 @@ func RegisterRoutes(m *web.Route) { }) m.Route("/delete", "GET,POST", org.SettingsDelete) + + m.Group("/packages", func() { + m.Get("", org.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", org.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", org.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleEditPost) + m.Get("/preview", org.PackagesRulePreview) + }) + }) + }) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable }) diff --git a/services/forms/package_form.go b/services/forms/package_form.go new file mode 100644 index 0000000000000..6c3ff52a9c01b --- /dev/null +++ b/services/forms/package_form.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package forms + +import ( + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/web/middleware" + + "gitea.com/go-chi/binding" +) + +type PackageCleanupRuleForm struct { + ID int64 + Enabled bool + Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + KeepCount int `binding:"In(0,1,5,10,25,50,100)"` + KeepPattern string `binding:"RegexPattern"` + RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` + RemovePattern string `binding:"RegexPattern"` + MatchFullName bool + Action string `binding:"Required;In(save,remove)"` +} + +func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index d23a481f279e4..e5c85656177d1 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -82,6 +82,10 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e return nil } +func ShouldBeSkipped(pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) { + return pv.LowerVersion == "latest", nil +} + // UpdateRepositoryNames updates the repository name property for all packages of the specific owner func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) diff --git a/services/packages/packages.go b/services/packages/packages.go index 96132eac0980d..0c72be0902c8b 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -352,13 +352,75 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro } // Cleanup removes expired package data -func Cleanup(unused context.Context, olderThan time.Duration) error { +func Cleanup(_ context.Context, olderThan time.Duration) error { ctx, committer, err := db.TxContext() if err != nil { return err } defer committer.Close() + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: do not remove '%s/%s'", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + } + } + return nil + }) + if err != nil { + log.Error("%#v", err) + return err + } + if err := container_service.Cleanup(ctx, olderThan); err != nil { return err } diff --git a/templates/org/settings/navbar.tmpl b/templates/org/settings/navbar.tmpl index e7cbb87344d1b..dfd2c024ffe8c 100644 --- a/templates/org/settings/navbar.tmpl +++ b/templates/org/settings/navbar.tmpl @@ -17,6 +17,9 @@ {{.locale.Tr "settings.applications"}} {{end}} + + {{.locale.Tr "packages.title"}} + {{.locale.Tr "org.settings.delete"}} diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl new file mode 100644 index 0000000000000..bb5d95e1072a2 --- /dev/null +++ b/templates/org/settings/packages.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_edit.tmpl b/templates/org/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 0000000000000..8c3725f4d7e9e --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_preview.tmpl b/templates/org/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 0000000000000..e0e4652c367eb --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,13 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/package/shared/cleanup_rules/edit.tmpl b/templates/package/shared/cleanup_rules/edit.tmpl new file mode 100644 index 0000000000000..f8525afb70b45 --- /dev/null +++ b/templates/package/shared/cleanup_rules/edit.tmpl @@ -0,0 +1,73 @@ +

{{if .IsEditRule}}{{.locale.Tr "packages.owner.settings.cleanuprules.edit"}}{{else}}{{.locale.Tr "packages.owner.settings.cleanuprules.add"}}{{end}}

+
+
+ {{.CsrfTokenHtml}} + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.title"}}

+
+ + +
+
+ + +

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern.container" | Safe}}

+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.remove.title"}}

+
+ + +
+
+ + +
+
+ {{if .IsEditRule}} + + + {{.locale.Tr "packages.owner.settings.cleanuprules.preview"}} + {{else}} + + {{end}} +
+
+
diff --git a/templates/package/shared/cleanup_rules/list.tmpl b/templates/package/shared/cleanup_rules/list.tmpl new file mode 100644 index 0000000000000..9475a4fabb989 --- /dev/null +++ b/templates/package/shared/cleanup_rules/list.tmpl @@ -0,0 +1,34 @@ +

+ {{.locale.Tr "packages.owner.settings.cleanuprules.title"}} +
+ {{.locale.Tr "packages.owner.settings.cleanuprules.add"}} +
+

+
+
+ {{range .CleanupRules}} +
+
+
+ {{svg "octicon-kebab-horizontal"}} +
+ {{$.locale.Tr "edit"}} + {{$.locale.Tr "packages.owner.settings.cleanuprules.preview"}} +
+
+
+ {{svg .Type.SVGName 36}} +
+ {{.Type.Name}} +
{{if .Enabled}}{{$.locale.Tr "enabled"}}{{else}}{{$.locale.Tr "disabled"}}{{end}}
+ {{if .KeepCount}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count"}}: {{if eq .KeepCount 1}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.1"}}{{else}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" .KeepCount}}{{end}}
{{end}} + {{if .KeepPattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern"}}: {{EllipsisString .KeepPattern 100}}
{{end}} + {{if .RemoveDays}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.days"}}: {{$.locale.Tr "tool.days" .RemoveDays}}
{{end}} + {{if .RemovePattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.pattern"}}: {{EllipsisString .RemovePattern 100}}
{{end}} +
+
+ {{else}} +
{{.locale.Tr "packages.owner.settings.cleanuprules.none"}}
+ {{end}} +
+
\ No newline at end of file diff --git a/templates/package/shared/cleanup_rules/preview.tmpl b/templates/package/shared/cleanup_rules/preview.tmpl new file mode 100644 index 0000000000000..c59ad67f7749c --- /dev/null +++ b/templates/package/shared/cleanup_rules/preview.tmpl @@ -0,0 +1,34 @@ +

{{.locale.Tr "packages.owner.settings.cleanuprules.preview"}}

+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.preview.overview" (len .VersionsToRemove)}}

+
+
+ + + + + + + + + + + + + {{range .VersionsToRemove}} + + + + + + + + + {{else}} + + + + {{end}} + +
{{.locale.Tr "admin.packages.type"}}{{.locale.Tr "admin.packages.name"}}{{.locale.Tr "admin.packages.version"}}{{.locale.Tr "admin.packages.creator"}}{{.locale.Tr "admin.packages.size"}}{{.locale.Tr "admin.packages.published"}}
{{.Package.Type.Name}}{{.Package.Name}}{{.Version.Version}}{{.Creator.Name}}{{FileSize .CalculateBlobSize}}
{{.locale.Tr "packages.owner.settings.cleanuprules.preview.none"}}
+
diff --git a/templates/user/settings/navbar.tmpl b/templates/user/settings/navbar.tmpl index 01ae055d79097..84a74514bcbe9 100644 --- a/templates/user/settings/navbar.tmpl +++ b/templates/user/settings/navbar.tmpl @@ -18,6 +18,9 @@ {{.locale.Tr "settings.ssh_gpg_keys"}} + + {{.locale.Tr "packages.title"}} + {{.locale.Tr "settings.organization"}} diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl new file mode 100644 index 0000000000000..2612313454e88 --- /dev/null +++ b/templates/user/settings/packages.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_edit.tmpl b/templates/user/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 0000000000000..4cf642b7e166a --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_preview.tmpl b/templates/user/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 0000000000000..20041f9a42fdc --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,8 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+{{template "base/footer" .}} From 5a944f75a208226269173360293197a6c884885d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 08:48:59 +0000 Subject: [PATCH 02/10] Do not use Sleep. --- tests/integration/api_packages_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 25f5b3f2a12da..43c8d45977f1a 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -169,19 +169,19 @@ func TestPackageAccess(t *testing.T) { func TestPackageCleanup(t *testing.T) { defer tests.PrepareTestEnv(t)() - time.Sleep(time.Second) + duration, _ := time.ParseDuration("-1h") - pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0)) + pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) assert.NoError(t, err) assert.NotEmpty(t, pbs) _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_service.Cleanup(nil, time.Duration(0)) + err = packages_service.Cleanup(nil, duration) assert.NoError(t, err) - pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0)) + pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) assert.NoError(t, err) assert.Empty(t, pbs) From 9912c73e24a03b401d08383ad6ffb8b9bb398144 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 10:18:05 +0000 Subject: [PATCH 03/10] Add tests. --- models/migrations/v232.go | 22 ++-- tests/integration/api_packages_test.go | 173 +++++++++++++++++++++++-- 2 files changed, 172 insertions(+), 23 deletions(-) diff --git a/models/migrations/v232.go b/models/migrations/v232.go index 894703c43e4f3..25f3451eb9360 100644 --- a/models/migrations/v232.go +++ b/models/migrations/v232.go @@ -12,17 +12,17 @@ import ( func createPackageCleanupRuleTable(x *xorm.Engine) error { type PackageCleanupRule struct { - ID int64 `xorm:"pk autoincr"` - Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` - OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` - Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` - KeepCount int `xorm:"NOT NULL DEFAULT 0"` - KeepPattern string `xorm:"NOT NULL DEFAULT ''"` - RemoveDays int `xorm:"NOT NULL DEFAULT 0"` - RemovePattern string `xorm:"NOT NULL DEFAULT ''"` - MatchFullName bool `xorm:"NOT NULL DEFAULT false"` - CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` } return x.Sync2(new(PackageCleanupRule)) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 43c8d45977f1a..65f2d57ead587 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -171,20 +171,169 @@ func TestPackageCleanup(t *testing.T) { duration, _ := time.ParseDuration("-1h") - pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) - assert.NoError(t, err) - assert.NotEmpty(t, pbs) + t.Run("Common", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) + assert.NoError(t, err) + assert.NotEmpty(t, pbs) - _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) - assert.NoError(t, err) + _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) + assert.NoError(t, err) - err = packages_service.Cleanup(nil, duration) - assert.NoError(t, err) + err = packages_service.Cleanup(nil, duration) + assert.NoError(t, err) - pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) - assert.NoError(t, err) - assert.Empty(t, pbs) + pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) + assert.NoError(t, err) + assert.Empty(t, pbs) + + _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) + assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + }) - _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) - assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + t.Run("CleanupRules", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + type version struct { + Version string + ShouldExist bool + Created int64 + } + + cases := []struct { + Name string + Versions []version + Rule *packages_model.PackageCleanupRule + }{ + { + Name: "Disabled", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: false, + }, + }, + { + Name: "KeepCount", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: true}, + {Version: "test-3", ShouldExist: false, Created: 1}, + {Version: "test-4", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepCount: 2, + }, + }, + { + Name: "KeepPattern", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepPattern: "k.+p", + }, + }, + { + Name: "RemoveDays", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemoveDays: 60, + }, + }, + { + Name: "RemovePattern", + Versions: []version{ + {Version: "test", ShouldExist: true}, + {Version: "test-3", ShouldExist: false}, + {Version: "test-4", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemovePattern: `t[e]+st-\d+`, + }, + }, + { + Name: "MatchFullName", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "test", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemovePattern: `package/test|different/keep`, + MatchFullName: true, + }, + }, + { + Name: "Mixed", + Versions: []version{ + {Version: "keep", ShouldExist: true, Created: time.Now().Add(time.Duration(10000)).Unix()}, + {Version: "dummy", ShouldExist: true, Created: 1}, + {Version: "test-3", ShouldExist: true}, + {Version: "test-4", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepCount: 1, + KeepPattern: `dummy`, + RemoveDays: 7, + RemovePattern: `t[e]+st-\d+`, + }, + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + for _, v := range c.Versions { + url := fmt.Sprintf("/api/packages/%s/generic/package/%s/file.bin", user.Name, v.Version) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1})) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + if v.Created != 0 { + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version) + assert.NoError(t, err) + _, err = db.GetEngine(db.DefaultContext).Exec("UPDATE package_version SET created_unix = ? WHERE id = ?", v.Created, pv.ID) + assert.NoError(t, err) + } + } + + c.Rule.OwnerID = user.ID + c.Rule.Type = packages_model.TypeGeneric + + pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) + assert.NoError(t, err) + + err = packages_service.Cleanup(nil, duration) + assert.NoError(t, err) + + for _, v := range c.Versions { + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version) + if v.ShouldExist { + assert.NoError(t, err) + err = packages_service.DeletePackageVersionAndReferences(db.DefaultContext, pv) + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + } + } + + assert.NoError(t, packages_model.DeleteCleanupRuleByID(db.DefaultContext, pcr.ID)) + }) + } + }) } From a28caf9ea585602271c08fe06505e3cb49a270f9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 19:47:13 +0000 Subject: [PATCH 04/10] Add documentation. --- docs/content/doc/packages/storage.en-us.md | 84 ++++++++++++++++++++++ services/packages/packages.go | 11 ++- 2 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 docs/content/doc/packages/storage.en-us.md diff --git a/docs/content/doc/packages/storage.en-us.md b/docs/content/doc/packages/storage.en-us.md new file mode 100644 index 0000000000000..c922496a9902a --- /dev/null +++ b/docs/content/doc/packages/storage.en-us.md @@ -0,0 +1,84 @@ +--- +date: "2022-11-01T00:00:00+00:00" +title: "Storage" +slug: "packages/storage" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "storage" + weight: 5 + identifier: "storage" +--- + +# Storage + +This document describes the storage of the package registry and how it can be managed. + +**Table of Contents** + +{{< toc >}} + +## Deduplication + +The package registry has a build-in deduplication of uploaded blobs. +If two identical files are uploaded only one blob is saved on the filesystem. +This ensures no space is wasted for duplicated files. + +If two packages are uploaded with identical files, both packages will display the same size but on the filesystem they require only half of the size. +Whenever a package gets deleted only the references to the underlaying blobs are removed. +The blobs get not removed at this moment, so they still require space on the filesystem. +When a new package gets uploaded the existing blobs may get referenced again. + +These unreferenced blobs get deleted by a [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}). +The config setting `OLDER_THAN` configures how long unreferenced blobs are kept before they get deleted. + +## Cleanup Rules + +Package registries can become large over time without cleanup. +It's recommended to delete unnecessary packages and set up cleanup rules to automatically manage the package registry usage. +Every package owner (user or organization) manages the cleanup rules which are applied to their packages. + +|Setting|Description| +|-|-| +|Enabled|Turn the cleanup rule on or off.| +|Type|Every rule manages a specific package type.| +|Apply pattern to full package name|If enabled, the patterns below are applied to the full package name (`package/version`). Otherwise only the version (`version`) is used.| +|Keep the most recent|How many versions to *always* keep for each package.| +|Keep versions matching|The regex pattern that determines which versions to keep. An empty pattern keeps no version while `.+` keeps all versions. The container registry will always keep the `latest` version even if not configured.| +|Remove versions older than|Remove only versions older than the selected days.| +|Remove versions matching|The regex pattern that determines which versions to remove. An empty pattern or `.+` leads to the removal of every package if no other setting tells otherwise.| + +Every cleanup rule can show a preview of the affected packages. +This can be used to check if the cleanup rules is proper configured. + +### Regex examples + +Regex patterns are automatically surrounded with `\A` and `\z` anchors. +Do not include any `\A`, `\z`, `^` or `$` token in the regex patterns as they are not necessary. +The patterns are case-insensitive which matches the behaviour of the package registry in Gitea. + +|Pattern|Description| +|-|-| +|`.*`|Match every possible version.| +|`v.+`|Match versions that start with `v`.| +|`release`|Match only the version `release`.| +|`release.*`|Match versions that are either named or start with `release`.| +|`.+-temp-.+`|Match versions that contain `-temp-`.| +|`v.+\|release`|Match versions that either start with `v` or are named `release`.| +|`package/v.+\|other/release`|Match versions of the package `package` that start with `v` or the version `release` of the package `other`. This needs the setting *Apply pattern to full package name* enabled.| + +### How the cleanup rules work + +The cleanup rules are part of the [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}) and run periodicly. + +The cleanup rule: + +1. Collects all packages of the package type for the owners registry. +1. For every package it collects all versions. +1. Excludes from the list the # versions based on the *Keep the most recent* value. +1. Excludes from the list any versions matching the *Keep versions matching* value. +1. Excludes from the list the versions more recent than the *Remove versions older than* value. +1. Excludes from the list any versions not matching the *Remove versions matching* value. +1. Deletes the remaining versions. diff --git a/services/packages/packages.go b/services/packages/packages.go index 0c72be0902c8b..52e4ffafca8c2 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -352,7 +352,7 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro } // Cleanup removes expired package data -func Cleanup(_ context.Context, olderThan time.Duration) error { +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { ctx, committer, err := db.TxContext() if err != nil { return err @@ -360,6 +360,12 @@ func Cleanup(_ context.Context, olderThan time.Duration) error { defer committer.Close() err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-taskCtx.Done(): + return db.ErrCancelled{"While processing package cleanup rules"} + default: + } + if err := pcr.CompiledPattern(); err != nil { return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) } @@ -403,7 +409,7 @@ func Cleanup(_ context.Context, olderThan time.Duration) error { continue } if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: do not remove '%s/%s'", pcr.ID, p.Name, pv.Version) + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) continue } @@ -417,7 +423,6 @@ func Cleanup(_ context.Context, olderThan time.Duration) error { return nil }) if err != nil { - log.Error("%#v", err) return err } From 8ec171737c4aedcd9f91cc52b96bb36a5d5ac08e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 19:54:55 +0000 Subject: [PATCH 05/10] Check if packages are enabled. --- routers/web/web.go | 23 +++++++++++++++-------- templates/org/settings/navbar.tmpl | 2 ++ templates/user/settings/navbar.tmpl | 2 ++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index 4aff8354982c0..707346b89043a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -303,6 +303,13 @@ func RegisterRoutes(m *web.Route) { } } + packagesEnabled := func(ctx *context.Context) { + if !setting.Packages.Enabled { + ctx.Error(http.StatusForbidden) + return + } + } + // FIXME: not all routes need go through same middleware. // Especially some AJAX requests, we can reduce middleware number to improve performance. // Routers. @@ -456,13 +463,14 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", user_setting.PackagesRulePreview) }) }) - }) + }, packagesEnabled) m.Get("/organization", user_setting.Organization) m.Get("/repos", user_setting.Repos) m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository) }, reqSignIn, func(ctx *context.Context) { ctx.Data["PageIsUserSettings"] = true ctx.Data["AllThemes"] = setting.UI.Themes + ctx.Data["EnablePackages"] = setting.Packages.Enabled }) m.Group("/user", func() { @@ -540,12 +548,10 @@ func RegisterRoutes(m *web.Route) { m.Post("/delete", admin.DeleteRepo) }) - if setting.Packages.Enabled { - m.Group("/packages", func() { - m.Get("", admin.Packages) - m.Post("/delete", admin.DeletePackageVersion) - }) - } + m.Group("/packages", func() { + m.Get("", admin.Packages) + m.Post("/delete", admin.DeletePackageVersion) + }, packagesEnabled) m.Group("/hooks", func() { m.Get("", admin.DefaultOrSystemWebhooks) @@ -779,9 +785,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", org.PackagesRulePreview) }) }) - }) + }, packagesEnabled) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable + ctx.Data["EnablePackages"] = setting.Packages.Enabled }) }, context.OrgAssignment(true, true)) }, reqSignIn) diff --git a/templates/org/settings/navbar.tmpl b/templates/org/settings/navbar.tmpl index dfd2c024ffe8c..7df1c85903c9c 100644 --- a/templates/org/settings/navbar.tmpl +++ b/templates/org/settings/navbar.tmpl @@ -17,9 +17,11 @@ {{.locale.Tr "settings.applications"}} {{end}} + {{if .EnablePackages}} {{.locale.Tr "packages.title"}} + {{end}} {{.locale.Tr "org.settings.delete"}} diff --git a/templates/user/settings/navbar.tmpl b/templates/user/settings/navbar.tmpl index 84a74514bcbe9..d17494fc04856 100644 --- a/templates/user/settings/navbar.tmpl +++ b/templates/user/settings/navbar.tmpl @@ -18,9 +18,11 @@ {{.locale.Tr "settings.ssh_gpg_keys"}} + {{if .EnablePackages}} {{.locale.Tr "packages.title"}} + {{end}} {{.locale.Tr "settings.organization"}} From 88f99adc029be56f9bc19b07acfd2e16ca77db8f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 21:09:16 +0000 Subject: [PATCH 06/10] lint --- routers/web/shared/packages/packages.go | 6 ------ services/packages/packages.go | 2 +- templates/package/shared/cleanup_rules/list.tmpl | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go index 82cb4d37c6a7d..02702fee2fe44 100644 --- a/routers/web/shared/packages/packages.go +++ b/routers/web/shared/packages/packages.go @@ -20,12 +20,6 @@ import ( container_service "code.gitea.io/gitea/services/packages/container" ) -const ( - tplSettingsPackages base.TplName = "user/settings/packages" - tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" - tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" -) - func SetPackagesContext(ctx *context.Context, owner *user_model.User) { pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID) if err != nil { diff --git a/services/packages/packages.go b/services/packages/packages.go index 52e4ffafca8c2..dc17b185f980f 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -362,7 +362,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { select { case <-taskCtx.Done(): - return db.ErrCancelled{"While processing package cleanup rules"} + return db.ErrCancelledf("While processing package cleanup rules") default: } diff --git a/templates/package/shared/cleanup_rules/list.tmpl b/templates/package/shared/cleanup_rules/list.tmpl index 9475a4fabb989..09f95e4f4a4bd 100644 --- a/templates/package/shared/cleanup_rules/list.tmpl +++ b/templates/package/shared/cleanup_rules/list.tmpl @@ -31,4 +31,4 @@
{{.locale.Tr "packages.owner.settings.cleanuprules.none"}}
{{end}} - \ No newline at end of file + From db92e67aefa6937e9216dc36290b0c667634c5d3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 1 Nov 2022 22:14:47 +0000 Subject: [PATCH 07/10] Provide context. --- tests/integration/api_packages_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 65f2d57ead587..d2fdfb768017e 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -181,7 +181,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_service.Cleanup(nil, duration) + err = packages_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -318,7 +318,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_service.Cleanup(nil, duration) + err = packages_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions { From 7e9764d6b7344ff82dcd59153c26d3342ead50f4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 3 Nov 2022 15:38:42 +0000 Subject: [PATCH 08/10] Handle multiarch manifests. --- models/packages/package_version.go | 9 ++++++++ routers/web/shared/packages/packages.go | 2 +- services/packages/container/cleanup.go | 29 +++++++++++++++++++++++-- services/packages/packages.go | 2 +- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 782261c575dcf..1c0be52680ef6 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P count, err := sess.FindAndCount(&pvs) return pvs, count, err } + +// ExistVersion checks if a version matching the search options exist +func ExistVersion(ctx context.Context, opts *PackageSearchOptions) (bool, error) { + return db.GetEngine(ctx). + Where(opts.toConds()). + Table("package_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Exist(new(PackageVersion)) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go index 02702fee2fe44..5e934d707ee41 100644 --- a/routers/web/shared/packages/packages.go +++ b/routers/web/shared/packages/packages.go @@ -165,7 +165,7 @@ func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) { return } for _, pv := range pvs { - if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { ctx.ServerError("ShouldBeSkipped", err) return } else if skip { diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index e5c85656177d1..2ee2017e129ca 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -13,6 +13,7 @@ import ( container_model "code.gitea.io/gitea/models/packages/container" user_model "code.gitea.io/gitea/models/user" container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/util" ) @@ -82,8 +83,32 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e return nil } -func ShouldBeSkipped(pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) { - return pv.LowerVersion == "latest", nil +func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) { + // Always skip the "latest" tag + if pv.LowerVersion == "latest" { + return true, nil + } + + // Check if the version is a digest (or untagged) + if oci.Digest(pv.LowerVersion).Validate() { + // Check if there is another manifest referencing this version + has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + Properties: map[string]string{ + container_module.PropertyManifestReference: pv.LowerVersion, + }, + }) + if err != nil { + return false, err + } + + // Skip it if the version is referenced + if has { + return true, nil + } + } + + return false, nil } // UpdateRepositoryNames updates the repository name property for all packages of the specific owner diff --git a/services/packages/packages.go b/services/packages/packages.go index dc17b185f980f..7acc712c7731a 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -388,7 +388,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) } for _, pv := range pvs { - if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) } else if skip { log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) From 5ff0a07aaa050db6364f21968a547cef8680edb4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 3 Nov 2022 15:41:25 +0000 Subject: [PATCH 09/10] Split file. --- services/packages/container/cleanup.go | 24 ----------------- services/packages/container/common.go | 36 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 services/packages/container/common.go diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index 2ee2017e129ca..e3d414d45c4e9 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -6,12 +6,10 @@ package container import ( "context" - "strings" "time" packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" - user_model "code.gitea.io/gitea/models/user" container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/util" @@ -110,25 +108,3 @@ func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule return false, nil } - -// UpdateRepositoryNames updates the repository name property for all packages of the specific owner -func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { - ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) - if err != nil { - return err - } - - newOwnerName = strings.ToLower(newOwnerName) - - for _, p := range ps { - if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { - return err - } - - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil { - return err - } - } - - return nil -} diff --git a/services/packages/container/common.go b/services/packages/container/common.go new file mode 100644 index 0000000000000..40d8914a01635 --- /dev/null +++ b/services/packages/container/common.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + container_module "code.gitea.io/gitea/modules/packages/container" +) + +// UpdateRepositoryNames updates the repository name property for all packages of the specific owner +func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { + ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) + if err != nil { + return err + } + + newOwnerName = strings.ToLower(newOwnerName) + + for _, p := range ps { + if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { + return err + } + + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil { + return err + } + } + + return nil +} From f2b3bb75f7c491b769157dda3f60d28e2fc3de61 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 9 Nov 2022 16:33:13 +0000 Subject: [PATCH 10/10] Use db.Insert. --- models/packages/package_cleanup_rule.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/packages/package_cleanup_rule.go b/models/packages/package_cleanup_rule.go index dbed7545b84e7..ab45226cf1503 100644 --- a/models/packages/package_cleanup_rule.go +++ b/models/packages/package_cleanup_rule.go @@ -64,8 +64,7 @@ func (pcr *PackageCleanupRule) CompiledPattern() error { } func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) { - _, err := db.GetEngine(ctx).Insert(pcr) - return pcr, err + return pcr, db.Insert(ctx, pcr) } func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) {