diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ffc4e406780e7..fd71cdf0de50b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -928,6 +928,15 @@ UPDATE_EXISTING = true ; Interval as a duration between each synchronization. (default every 24h) SCHEDULE = @every 24h +; Prune hook_task table +[cron.prune_hook_task_table] +; Whether to enable the job +ENABLED = true +; Whether to always run at start up time (if ENABLED) +RUN_AT_START = false +; Time interval for job to run +SCHEDULE = @every 24h + [git] ; The path of git executable. If empty, Gitea searches through the PATH environment. PATH = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 1e48ee2597f76..7112ee6c703a8 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -598,6 +598,12 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false` - `SCHEDULE`: **@every 24h** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts. +### Cron - Prune hook_task Table (`cron.prune_hook_task_table`) + +- `ENABLED`: **true**: Enable service. In the repository table, there are two columns to additionally override the behavior per repository: is_hook_task_purge_enabled - enable or disable purging of hook_task data by repository and number_webhook_deliveries_to_keep - the process will leave the most recently delivered webhooks, this value controls how many are kept. These settings can also be modified per repository in the Gitea UI. By default, the process is enabled for every repository and 10 webhooks are kept. +- `RUN_AT_START`: **false**: Run prune hook_task at start time (if ENABLED). +- `SCHEDULE`: **@every 24h**: Cron syntax for pruning hook_task table. + ## Git (`git`) - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment. diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index ea1bf596492f6..5c1e484c8e3bf 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -239,6 +239,8 @@ var migrations = []Migration{ NewMigration("set default password algorithm to Argon2", setDefaultPasswordToArgon2), // v152 -> v153 NewMigration("add TrustModel field to Repository", addTrustModelToRepository), + // v153 -> v154 + NewMigration("Add IsHookTaskPurgeEnabled and NumberWebhookDeliveriesToKeep columns to Repository table", addHookTaskPurge), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v153.go b/models/migrations/v153.go new file mode 100644 index 0000000000000..a0c09efcaa84c --- /dev/null +++ b/models/migrations/v153.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" +) + +func addHookTaskPurge(x *xorm.Engine) error { + type Repository struct { + ID int64 `xorm:"pk autoincr"` + IsHookTaskPurgeEnabled bool `xorm:"NOT NULL DEFAULT true"` + NumberWebhookDeliveriesToKeep int64 `xorm:"NOT NULL DEFAULT 10"` + } + + if err := x.Sync2(new(Repository)); err != nil { + return err + } + + _, err := x.Exec("UPDATE repository SET is_hook_task_purge_enabled = ?, number_webhook_deliveries_to_keep = ?", + setting.Repository.DefaultIsHookTaskPurgeEnabled, setting.Repository.DefaultNumberWebhookDeliveriesToKeep) + return err +} diff --git a/models/repo.go b/models/repo.go index 25fe3f63d39c9..f55faaa0c2e2c 100644 --- a/models/repo.go +++ b/models/repo.go @@ -237,6 +237,8 @@ type Repository struct { StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"` + IsHookTaskPurgeEnabled bool `xorm:"NOT NULL DEFAULT true"` + NumberWebhookDeliveriesToKeep int64 `xorm:"NOT NULL DEFAULT 10"` Topics []string `xorm:"TEXT JSON"` TrustModel TrustModelType diff --git a/models/webhook.go b/models/webhook.go index 54cd9b6565841..667409f0e0801 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -798,3 +798,31 @@ func FindRepoUndeliveredHookTasks(repoID int64) ([]*HookTask, error) { } return tasks, nil } + +// DeleteDeliveredHookTasks deletes delivered hook tasks of one repository, leaving the most recent delivered based on the parameter +func DeleteDeliveredHookTasks(repoID int64, numberDeliveriesToKeep int64) error { + var deliveryDates = make([]int64, 0, 10) + err := x.Table("hook_task"). + Where("hook_task.repo_id = ? AND hook_task.is_delivered = ?", repoID, true). + Cols("hook_task.delivered"). + Join("INNER", "repository", "hook_task.repo_id = repository.id"). + And("repository.is_hook_task_purge_enabled = ?", true). + OrderBy("hook_task.delivered desc"). + Limit(1, int(numberDeliveriesToKeep)). + Find(&deliveryDates) + if err != nil { + return err + } + + if len(deliveryDates) > 0 { + deletes, err := x. + Where("repo_id = ? and is_delivered = ? and delivered <= ?", repoID, true, deliveryDates[0]). + Delete(new(HookTask)) + if err != nil { + return err + } + log.Trace("From repo %d deleted in total %d from hook_task", repoID, deletes) + } + + return nil +} diff --git a/models/webhook_test.go b/models/webhook_test.go index 5ee7f9159bd50..6f4066ba1453e 100644 --- a/models/webhook_test.go +++ b/models/webhook_test.go @@ -7,6 +7,7 @@ package models import ( "encoding/json" "testing" + "time" api "code.gitea.io/gitea/modules/structs" @@ -245,3 +246,58 @@ func TestUpdateHookTask(t *testing.T) { assert.NoError(t, UpdateHookTask(hook)) AssertExistsAndLoadBean(t, hook) } + +func TestDeleteDeliveredHookTasks_DeletesDelivered(t *testing.T) { + assert.NoError(t, PrepareTestDatabase()) + hookTask := &HookTask{ + RepoID: 3, + HookID: 3, + Type: GITEA, + URL: "http://www.example.com/unit_test", + Payloader: &api.PushPayload{}, + IsDelivered: true, + } + AssertNotExistsBean(t, hookTask) + assert.NoError(t, CreateHookTask(hookTask)) + AssertExistsAndLoadBean(t, hookTask) + + assert.NoError(t, DeleteDeliveredHookTasks(3, 0)) + AssertNotExistsBean(t, hookTask) +} + +func TestDeleteDeliveredHookTasks_LeavesUndelivered(t *testing.T) { + assert.NoError(t, PrepareTestDatabase()) + hookTask := &HookTask{ + RepoID: 2, + HookID: 4, + Type: GITEA, + URL: "http://www.example.com/unit_test", + Payloader: &api.PushPayload{}, + IsDelivered: false, + } + AssertNotExistsBean(t, hookTask) + assert.NoError(t, CreateHookTask(hookTask)) + AssertExistsAndLoadBean(t, hookTask) + + assert.NoError(t, DeleteDeliveredHookTasks(3, 0)) + AssertExistsAndLoadBean(t, hookTask) +} + +func TestDeleteDeliveredHookTasks_LeavesMostRecentTask(t *testing.T) { + assert.NoError(t, PrepareTestDatabase()) + hookTask := &HookTask{ + RepoID: 2, + HookID: 4, + Type: GITEA, + URL: "http://www.example.com/unit_test", + Payloader: &api.PushPayload{}, + IsDelivered: true, + Delivered: time.Now().UnixNano(), + } + AssertNotExistsBean(t, hookTask) + assert.NoError(t, CreateHookTask(hookTask)) + AssertExistsAndLoadBean(t, hookTask) + + assert.NoError(t, DeleteDeliveredHookTasks(3, 1)) + AssertExistsAndLoadBean(t, hookTask) +} diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index f1130f372b794..a0011da206675 100644 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -149,6 +149,8 @@ type RepoSettingForm struct { // Admin settings EnableHealthCheck bool EnableCloseIssuesViaCommitInAnyBranch bool + IsHookTaskPurgeEnabled bool + NumberWebhookDeliveriesToKeep int64 } // Validate validates the fields diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 4da21fc7d9ddd..8dbeca328be45 100644 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -108,6 +108,16 @@ func registerUpdateMigrationPosterID() { }) } +func registerPrunHookTaskTable() { + RegisterTaskFatal("prune_hook_task_table", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 24h", + }, func(ctx context.Context, _ *models.User, _ Config) error { + return repository_service.PruneHookTaskTable(ctx) + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -116,4 +126,5 @@ func initBasicTasks() { registerSyncExternalUsers() registerDeletedBranchesCleanup() registerUpdateMigrationPosterID() + registerPrunHookTaskTable() } diff --git a/modules/repository/create.go b/modules/repository/create.go index c180b9b9481d7..dba5c680e1ceb 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -39,6 +39,8 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + IsHookTaskPurgeEnabled: setting.Repository.DefaultIsHookTaskPurgeEnabled, + NumberWebhookDeliveriesToKeep: setting.Repository.DefaultNumberWebhookDeliveriesToKeep, Status: opts.Status, IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, diff --git a/modules/repository/prune_hook_task.go b/modules/repository/prune_hook_task.go new file mode 100644 index 0000000000000..db511db9bb5cd --- /dev/null +++ b/modules/repository/prune_hook_task.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repository + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "xorm.io/builder" +) + +// PruneHookTaskTable deletes rows from hook_task as needed. +func PruneHookTaskTable(ctx context.Context) error { + log.Trace("Doing: PruneHookTaskTable") + + if err := models.Iterate( + models.DefaultDBContext(), + new(models.Repository), + builder.Expr("id>0 AND is_hook_task_purge_enabled=?", true), + func(idx int, bean interface{}) error { + select { + case <-ctx.Done(): + return fmt.Errorf("Aborted due to shutdown") + default: + } + repo := bean.(*models.Repository) + repoPath := repo.RepoPath() + log.Trace("Running prune hook_task table on repository %s", repoPath) + if err := models.DeleteDeliveredHookTasks(repo.ID, repo.NumberWebhookDeliveriesToKeep); err != nil { + desc := fmt.Sprintf("Failed to prune hook_task on repository (%s): %v", repoPath, err) + log.Warn(desc) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } + return nil + }, + ); err != nil { + return err + } + + log.Trace("Finished: PruneHookTaskTable") + return nil +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 67dd80535338d..194320800a33e 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -37,6 +37,8 @@ var ( AccessControlAllowOrigin string UseCompatSSHURI bool DefaultCloseIssuesViaCommitsInAnyBranch bool + DefaultIsHookTaskPurgeEnabled bool + DefaultNumberWebhookDeliveriesToKeep int64 EnablePushCreateUser bool EnablePushCreateOrg bool DisabledRepoUnits []string @@ -140,6 +142,8 @@ var ( AccessControlAllowOrigin: "", UseCompatSSHURI: false, DefaultCloseIssuesViaCommitsInAnyBranch: false, + DefaultIsHookTaskPurgeEnabled: true, + DefaultNumberWebhookDeliveriesToKeep: 10, EnablePushCreateUser: false, EnablePushCreateOrg: false, DisabledRepoUnits: []string{}, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ea76d4cb6a7c5..168ca1d12617e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1447,6 +1447,8 @@ settings.projects_desc = Enable Repository Projects settings.admin_settings = Administrator Settings settings.admin_enable_health_check = Enable Repository Health Checks (git fsck) settings.admin_enable_close_issues_via_commit_in_any_branch = Close an issue via a commit made in a non default branch +settings.admin_enable_hook_task_purge_1 = Purge webhook delivery notifications +settings.admin_enable_hook_task_purge_2 = Webhook deliveries to keep settings.danger_zone = Danger Zone settings.new_owner_has_same_repo = The new owner already has a repository with same name. Please choose another name. settings.convert = Convert to Regular Repository @@ -1982,6 +1984,7 @@ dashboard.resync_all_sshkeys.desc = (Not needed for the built-in SSH server.) dashboard.resync_all_hooks = Resynchronize pre-receive, update and post-receive hooks of all repositories. dashboard.reinit_missing_repos = Reinitialize all missing Git repositories for which records exist dashboard.sync_external_users = Synchronize external user data +dashboard.prune_hook_task_table = Prune hook_task table. dashboard.server_uptime = Server Uptime dashboard.current_goroutine = Current Goroutines dashboard.current_memory_usage = Current Memory Usage diff --git a/routers/repo/setting.go b/routers/repo/setting.go index d2c20fb03a6bf..ad4917b1235cf 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -357,6 +357,14 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { repo.CloseIssuesViaCommitInAnyBranch = form.EnableCloseIssuesViaCommitInAnyBranch } + if repo.IsHookTaskPurgeEnabled != form.IsHookTaskPurgeEnabled { + repo.IsHookTaskPurgeEnabled = form.IsHookTaskPurgeEnabled + } + + if form.IsHookTaskPurgeEnabled && repo.NumberWebhookDeliveriesToKeep != form.NumberWebhookDeliveriesToKeep { + repo.NumberWebhookDeliveriesToKeep = form.NumberWebhookDeliveriesToKeep + } + if err := models.UpdateRepository(repo, false); err != nil { ctx.ServerError("UpdateRepository", err) return diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index a8e050c58347c..ed1c31245637a 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -400,11 +400,22 @@ -
- - +
+
+ + +
+
+
+
+ + +
+
+
+ +
-
diff --git a/web_src/js/index.js b/web_src/js/index.js index 415db385b39e6..2914718389643 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -743,6 +743,17 @@ async function initRepository() { if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled'); } }); + + // Enable number of webhooks to keep. + $('.enable-webhooks-to-keep').change(function () { + if (this.checked) { + $($(this).data('target')).removeClass('disabled'); + if (!$(this).data('context')) $($(this).data('context')).addClass('disabled'); + } else { + $($(this).data('target')).addClass('disabled'); + if (!$(this).data('context')) $($(this).data('context')).removeClass('disabled'); + } + }); } // Labels