From cf8476a2e1b85ab011ab53b0b86c299006b78a9e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 Sep 2023 22:18:50 +0200 Subject: [PATCH 1/9] Add doctor dbconsistency fix to delete repos with no owner --- modules/doctor/dbconsistency.go | 6 +++++ modules/doctor/repository.go | 47 +++++++++++++++++++++++++++++++++ services/repository/delete.go | 10 ++++--- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 modules/doctor/repository.go diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go index 548687355605b..42cab1f41e900 100644 --- a/modules/doctor/dbconsistency.go +++ b/modules/doctor/dbconsistency.go @@ -151,6 +151,12 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er Fixer: activities_model.FixActionCreatedUnixString, FixedMessage: "Set to zero", }, + { + Name: "Repos with no existing owner", + Counter: CountOrphanedRepos, + Fixer: DeleteOrphanedRepos, + FixedMessage: "Deleted all related stuff of orphaned repos", + }, } // TODO: function to recalc all counters diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go new file mode 100644 index 0000000000000..50d9d1657ace4 --- /dev/null +++ b/modules/doctor/repository.go @@ -0,0 +1,47 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package doctor + +import ( + "context" + + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" + repo_service "code.gitea.io/gitea/services/repository" + + "xorm.io/builder" +) + +// CountOrphanedRepos count repository where user of owner_id do not exist +func CountOrphanedRepos(ctx context.Context) (int64, error) { + return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=user.id") +} + +// DeleteOrphanedRepos delete repository where user of owner_id do not exist +func DeleteOrphanedRepos(ctx context.Context) (int64, error) { + batchSize := db.MaxBatchInsertSize("repository") + e := db.GetEngine(ctx) + var deleted int64 + adminUser := &user_model.User{IsAdmin: true} + + for { + var ids []int64 + e.Table("`repository`"). + Join("LEFT", "`user`", "repository.owner_id=user.id"). + Where(builder.IsNull{"`user`.id"}). + Select("`repository`.id").Limit(batchSize).Get(&ids) + + // if we don't get ids we deleted them all + if len(ids) == 0 { + return deleted, nil + } + + for _, id := range ids { + if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, 0, id); err != nil { + return deleted, err + } + deleted++ + } + } +} diff --git a/services/repository/delete.go b/services/repository/delete.go index f3bf91af4f38c..be1ebbfb832aa 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -54,9 +54,11 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, r } // In case is a organization. - org, err := user_model.GetUserByID(ctx, uid) - if err != nil { - return err + var org *user_model.User + if uid != 0 { + if org, err = user_model.GetUserByID(ctx, uid); err != nil { + return err + } } repo := &repo_model.Repository{OwnerID: uid} @@ -95,7 +97,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, r } } - if org.IsOrganization() { + if org != nil && org.IsOrganization() { teams, err := organization.FindOrgTeams(ctx, org.ID) if err != nil { return err From 2b6ee3c404dc66b3c807a1bcb9e1b8e97a58eef6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 Sep 2023 22:23:57 +0200 Subject: [PATCH 2/9] Update modules/doctor/repository.go --- modules/doctor/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index 50d9d1657ace4..4b542ceb82a51 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. +// Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package doctor From debbd1607e8269bf77702b8c314d71422201b5b7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 Sep 2023 22:50:34 +0200 Subject: [PATCH 3/9] ... --- modules/doctor/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index 4b542ceb82a51..fc52e58114222 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -30,7 +30,7 @@ func DeleteOrphanedRepos(ctx context.Context) (int64, error) { e.Table("`repository`"). Join("LEFT", "`user`", "repository.owner_id=user.id"). Where(builder.IsNull{"`user`.id"}). - Select("`repository`.id").Limit(batchSize).Get(&ids) + Select("`repository`.id").Limit(batchSize).Find(&ids) // if we don't get ids we deleted them all if len(ids) == 0 { From 8c207596562de22a0805b547158cb7a00528a64a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 Sep 2023 23:01:13 +0200 Subject: [PATCH 4/9] check the error --- modules/doctor/repository.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index fc52e58114222..71e4788f9fd51 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -27,10 +27,12 @@ func DeleteOrphanedRepos(ctx context.Context) (int64, error) { for { var ids []int64 - e.Table("`repository`"). + if err := e.Table("`repository`"). Join("LEFT", "`user`", "repository.owner_id=user.id"). Where(builder.IsNull{"`user`.id"}). - Select("`repository`.id").Limit(batchSize).Find(&ids) + Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil { + return deleted, err + } // if we don't get ids we deleted them all if len(ids) == 0 { From cf00b1bc3b9b19edb12304b82046bf9e27c52129 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 30 Sep 2023 17:04:56 +0200 Subject: [PATCH 5/9] use dedicated optional arg instead --- modules/doctor/repository.go | 2 +- services/repository/delete.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index 71e4788f9fd51..a844165b600d1 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -40,7 +40,7 @@ func DeleteOrphanedRepos(ctx context.Context) (int64, error) { } for _, id := range ids { - if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, 0, id); err != nil { + if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, 0, id, true); err != nil { return deleted, err } deleted++ diff --git a/services/repository/delete.go b/services/repository/delete.go index be1ebbfb832aa..40bdc8b619f43 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -33,7 +33,7 @@ import ( // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, repoID int64) error { +func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -53,9 +53,10 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, r return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err) } - // In case is a organization. + // In case owner is a organization, we delete repo specific teams + // if ignoreOrgTeams is not true var org *user_model.User - if uid != 0 { + if len(ignoreOrgTeams) == 0 || !ignoreOrgTeams[0] { if org, err = user_model.GetUserByID(ctx, uid); err != nil { return err } From 55adbd1fcac0825d9c474fca64da1ba95eddbd44 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 30 Sep 2023 17:06:17 +0200 Subject: [PATCH 6/9] Update modules/doctor/repository.go --- modules/doctor/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index a844165b600d1..e673f7e12de1a 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -34,7 +34,7 @@ func DeleteOrphanedRepos(ctx context.Context) (int64, error) { return deleted, err } - // if we don't get ids we deleted them all + // if we don't get ids we have deleted them all if len(ids) == 0 { return deleted, nil } From abc89b4d90414379443a2f47f35c6a9cfc551043 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 30 Sep 2023 17:07:16 +0200 Subject: [PATCH 7/9] Update services/repository/delete.go --- services/repository/delete.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/delete.go b/services/repository/delete.go index 40bdc8b619f43..a06d8a803feb2 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -53,7 +53,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, uid, r return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err) } - // In case owner is a organization, we delete repo specific teams + // In case owner is a organization, we have to change repo specific teams // if ignoreOrgTeams is not true var org *user_model.User if len(ignoreOrgTeams) == 0 || !ignoreOrgTeams[0] { From 14629dd453453e2921eb82b9165c80fcfb8f43f2 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 2 Oct 2023 23:39:37 +0200 Subject: [PATCH 8/9] Update modules/doctor/dbconsistency.go Co-authored-by: silverwind --- modules/doctor/dbconsistency.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go index 69268c4ecc725..51c29fa148626 100644 --- a/modules/doctor/dbconsistency.go +++ b/modules/doctor/dbconsistency.go @@ -155,7 +155,7 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er Name: "Repos with no existing owner", Counter: CountOrphanedRepos, Fixer: DeleteOrphanedRepos, - FixedMessage: "Deleted all related stuff of orphaned repos", + FixedMessage: "Deleted all content related to orphaned repos", }, } From 834de9800a6256145596f6d62be478de23e57986 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Wed, 4 Oct 2023 04:25:03 +0200 Subject: [PATCH 9/9] move to its own task --- modules/doctor/dbconsistency.go | 6 ------ modules/doctor/repository.go | 29 +++++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go index 51c29fa148626..e5fc5785e826c 100644 --- a/modules/doctor/dbconsistency.go +++ b/modules/doctor/dbconsistency.go @@ -151,12 +151,6 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er Fixer: activities_model.FixActionCreatedUnixString, FixedMessage: "Set to zero", }, - { - Name: "Repos with no existing owner", - Counter: CountOrphanedRepos, - Fixer: DeleteOrphanedRepos, - FixedMessage: "Deleted all content related to orphaned repos", - }, } // TODO: function to recalc all counters diff --git a/modules/doctor/repository.go b/modules/doctor/repository.go index e673f7e12de1a..6569378cbe4be 100644 --- a/modules/doctor/repository.go +++ b/modules/doctor/repository.go @@ -8,18 +8,29 @@ import ( "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" repo_service "code.gitea.io/gitea/services/repository" "xorm.io/builder" ) -// CountOrphanedRepos count repository where user of owner_id do not exist -func CountOrphanedRepos(ctx context.Context) (int64, error) { +func handleDeleteOrphanedRepos(ctx context.Context, logger log.Logger, autofix bool) error { + test := &consistencyCheck{ + Name: "Repos with no existing owner", + Counter: countOrphanedRepos, + Fixer: deleteOrphanedRepos, + FixedMessage: "Deleted all content related to orphaned repos", + } + return test.Run(ctx, logger, autofix) +} + +// countOrphanedRepos count repository where user of owner_id do not exist +func countOrphanedRepos(ctx context.Context) (int64, error) { return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=user.id") } -// DeleteOrphanedRepos delete repository where user of owner_id do not exist -func DeleteOrphanedRepos(ctx context.Context) (int64, error) { +// deleteOrphanedRepos delete repository where user of owner_id do not exist +func deleteOrphanedRepos(ctx context.Context) (int64, error) { batchSize := db.MaxBatchInsertSize("repository") e := db.GetEngine(ctx) var deleted int64 @@ -47,3 +58,13 @@ func DeleteOrphanedRepos(ctx context.Context) (int64, error) { } } } + +func init() { + Register(&Check{ + Title: "Deleted all content related to orphaned repos", + Name: "delete-orphaned-repos", + IsDefault: false, + Run: handleDeleteOrphanedRepos, + Priority: 4, + }) +}