Skip to content

Commit 520eb57

Browse files
wxiaoguangwxiaoguang
authored
Use a separate admin page to show global stats, remove actions stat (#25062)
Before, Gitea shows the database table stats on the `admin dashboard` page. It has some problems: * `count(*)` is quite heavy. If tables have many records, this blocks loading the admin page blocks for a long time * Some users had even reported issues that they can't visit their admin page because this page causes blocking or `50x error (reverse proxy timeout)` * The `actions` stat is not useful. The table is simply too large. Does it really matter if it contains 1,000,000 rows or 9,999,999 rows? * The translation `admin.dashboard.statistic_info` is difficult to maintain. So, this PR uses a separate page to show the stats and removes the `actions` stat. ![image](https://github.com/go-gitea/gitea/assets/2114189/babf7c61-b93b-4a62-bfaa-22983636427e) ## ⚠️ BREAKING The `actions` Prometheus metrics collector has been removed for the reasons mentioned beforehand. Please do not rely on its output anymore.
1 parent 4486dd3 commit 520eb57

File tree

9 files changed

+57
-53
lines changed

9 files changed

+57
-53
lines changed

‎models/activities/statistic.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import (
2121
type Statistic struct {
2222
Counter struct {
2323
User, Org, PublicKey,
24-
Repo, Watch, Star, Action, Access,
24+
Repo, Watch, Star, Access,
2525
Issue, IssueClosed, IssueOpen,
2626
Comment, Oauth, Follow,
2727
Mirror, Release, AuthSource, Webhook,
@@ -55,7 +55,6 @@ func GetStatistic() (stats Statistic) {
5555
stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{})
5656
stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
5757
stats.Counter.Star, _ = e.Count(new(repo_model.Star))
58-
stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action))
5958
stats.Counter.Access, _ = e.Count(new(access_model.Access))
6059

6160
type IssueCount struct {
@@ -83,7 +82,7 @@ func GetStatistic() (stats Statistic) {
8382
Find(&stats.Counter.IssueByRepository)
8483
}
8584

86-
issueCounts := []IssueCount{}
85+
var issueCounts []IssueCount
8786

8887
_ = e.Select("COUNT(*) AS count, is_closed").Table("issue").GroupBy("is_closed").Find(&issueCounts)
8988
for _, c := range issueCounts {

‎models/db/context.go

-25
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99

1010
"xorm.io/builder"
1111
"xorm.io/xorm"
12-
"xorm.io/xorm/schemas"
1312
)
1413

1514
// DefaultContext is the default context to run xorm queries in
@@ -241,30 +240,6 @@ func TableName(bean interface{}) string {
241240
return x.TableName(bean)
242241
}
243242

244-
// EstimateCount returns an estimate of total number of rows in table
245-
func EstimateCount(ctx context.Context, bean interface{}) (int64, error) {
246-
e := GetEngine(ctx)
247-
e.Context(ctx)
248-
249-
var rows int64
250-
var err error
251-
tablename := TableName(bean)
252-
switch x.Dialect().URI().DBType {
253-
case schemas.MYSQL:
254-
_, err = e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows)
255-
case schemas.POSTGRES:
256-
// the table can live in multiple schemas of a postgres database
257-
// See https://wiki.postgresql.org/wiki/Count_estimate
258-
tablename = x.TableName(bean, true)
259-
_, err = e.Context(ctx).SQL("SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = ?::regclass;", tablename).Get(&rows)
260-
case schemas.MSSQL:
261-
_, err = e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows)
262-
default:
263-
return e.Context(ctx).Count(tablename)
264-
}
265-
return rows, err
266-
}
267-
268243
// InTransaction returns true if the engine is in a transaction otherwise return false
269244
func InTransaction(ctx context.Context) bool {
270245
_, ok := inTransaction(ctx)

‎modules/metrics/collector.go

-12
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ const namespace = "gitea_"
1818
// exposes gitea metrics for prometheus
1919
type Collector struct {
2020
Accesses *prometheus.Desc
21-
Actions *prometheus.Desc
2221
Attachments *prometheus.Desc
2322
BuildInfo *prometheus.Desc
2423
Comments *prometheus.Desc
@@ -56,11 +55,6 @@ func NewCollector() Collector {
5655
"Number of Accesses",
5756
nil, nil,
5857
),
59-
Actions: prometheus.NewDesc(
60-
namespace+"actions",
61-
"Number of Actions",
62-
nil, nil,
63-
),
6458
Attachments: prometheus.NewDesc(
6559
namespace+"attachments",
6660
"Number of Attachments",
@@ -207,7 +201,6 @@ func NewCollector() Collector {
207201
// Describe returns all possible prometheus.Desc
208202
func (c Collector) Describe(ch chan<- *prometheus.Desc) {
209203
ch <- c.Accesses
210-
ch <- c.Actions
211204
ch <- c.Attachments
212205
ch <- c.BuildInfo
213206
ch <- c.Comments
@@ -246,11 +239,6 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) {
246239
prometheus.GaugeValue,
247240
float64(stats.Counter.Access),
248241
)
249-
ch <- prometheus.MustNewConstMetric(
250-
c.Actions,
251-
prometheus.GaugeValue,
252-
float64(stats.Counter.Action),
253-
)
254242
ch <- prometheus.MustNewConstMetric(
255243
c.Attachments,
256244
prometheus.GaugeValue,

‎options/locale/locale_en-US.ini

+2-1
Original file line numberDiff line numberDiff line change
@@ -2619,7 +2619,6 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec
26192619
dashboard.statistic = Summary
26202620
dashboard.operations = Maintenance Operations
26212621
dashboard.system_status = System Status
2622-
dashboard.statistic_info = The Gitea database holds <b>%d</b> users, <b>%d</b> organizations, <b>%d</b> public keys, <b>%d</b> repositories, <b>%d</b> watches, <b>%d</b> stars, ~<b>%d</b> actions, <b>%d</b> accesses, <b>%d</b> issues, <b>%d</b> comments, <b>%d</b> social accounts, <b>%d</b> follows, <b>%d</b> mirrors, <b>%d</b> releases, <b>%d</b> authentication sources, <b>%d</b> webhooks, <b>%d</b> milestones, <b>%d</b> labels, <b>%d</b> hook tasks, <b>%d</b> teams, <b>%d</b> update tasks, <b>%d</b> attachments.
26232622
dashboard.operation_name = Operation Name
26242623
dashboard.operation_switch = Switch
26252624
dashboard.operation_run = Run
@@ -3060,6 +3059,8 @@ config.xorm_log_sql = Log SQL
30603059
config.get_setting_failed = Get setting %s failed
30613060
config.set_setting_failed = Set setting %s failed
30623061
3062+
monitor.stats = Stats
3063+
30633064
monitor.cron = Cron Tasks
30643065
monitor.name = Name
30653066
monitor.schedule = Schedule

‎routers/web/admin/admin.go

+30-2
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@ import (
88
"fmt"
99
"net/http"
1010
"runtime"
11+
"sort"
1112
"time"
1213

1314
activities_model "code.gitea.io/gitea/models/activities"
1415
"code.gitea.io/gitea/modules/base"
1516
"code.gitea.io/gitea/modules/context"
17+
"code.gitea.io/gitea/modules/json"
1618
"code.gitea.io/gitea/modules/setting"
1719
"code.gitea.io/gitea/modules/updatechecker"
1820
"code.gitea.io/gitea/modules/web"
@@ -26,6 +28,7 @@ const (
2628
tplQueue base.TplName = "admin/queue"
2729
tplStacktrace base.TplName = "admin/stacktrace"
2830
tplQueueManage base.TplName = "admin/queue_manage"
31+
tplStats base.TplName = "admin/stats"
2932
)
3033

3134
var sysStatus struct {
@@ -111,7 +114,6 @@ func updateSystemStatus() {
111114
func Dashboard(ctx *context.Context) {
112115
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
113116
ctx.Data["PageIsAdminDashboard"] = true
114-
ctx.Data["Stats"] = activities_model.GetStatistic()
115117
ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate()
116118
ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion()
117119
// FIXME: update periodically
@@ -126,7 +128,6 @@ func DashboardPost(ctx *context.Context) {
126128
form := web.GetForm(ctx).(*forms.AdminDashboardForm)
127129
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
128130
ctx.Data["PageIsAdminDashboard"] = true
129-
ctx.Data["Stats"] = activities_model.GetStatistic()
130131
updateSystemStatus()
131132
ctx.Data["SysStatus"] = sysStatus
132133

@@ -153,3 +154,30 @@ func CronTasks(ctx *context.Context) {
153154
ctx.Data["Entries"] = cron.ListTasks()
154155
ctx.HTML(http.StatusOK, tplCron)
155156
}
157+
158+
func MonitorStats(ctx *context.Context) {
159+
ctx.Data["Title"] = ctx.Tr("admin.monitor.stats")
160+
ctx.Data["PageIsAdminMonitorStats"] = true
161+
bs, err := json.Marshal(activities_model.GetStatistic().Counter)
162+
if err != nil {
163+
ctx.ServerError("MonitorStats", err)
164+
return
165+
}
166+
statsCounter := map[string]any{}
167+
err = json.Unmarshal(bs, &statsCounter)
168+
if err != nil {
169+
ctx.ServerError("MonitorStats", err)
170+
return
171+
}
172+
statsKeys := make([]string, 0, len(statsCounter))
173+
for k := range statsCounter {
174+
if statsCounter[k] == nil {
175+
continue
176+
}
177+
statsKeys = append(statsKeys, k)
178+
}
179+
sort.Strings(statsKeys)
180+
ctx.Data["StatsKeys"] = statsKeys
181+
ctx.Data["StatsCounter"] = statsCounter
182+
ctx.HTML(http.StatusOK, tplStats)
183+
}

‎routers/web/web.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -538,8 +538,8 @@ func registerRoutes(m *web.Route) {
538538

539539
// ***** START: Admin *****
540540
m.Group("/admin", func() {
541-
m.Get("", adminReq, admin.Dashboard)
542-
m.Post("", adminReq, web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
541+
m.Get("", admin.Dashboard)
542+
m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
543543

544544
m.Group("/config", func() {
545545
m.Get("", admin.Config)
@@ -548,6 +548,7 @@ func registerRoutes(m *web.Route) {
548548
})
549549

550550
m.Group("/monitor", func() {
551+
m.Get("/stats", admin.MonitorStats)
551552
m.Get("/cron", admin.CronTasks)
552553
m.Get("/stacktrace", admin.Stacktrace)
553554
m.Post("/stacktrace/cancel/{pid}", admin.StacktraceCancel)

‎templates/admin/dashboard.tmpl

-8
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,6 @@
55
<p>{{(.locale.Tr "admin.dashboard.new_version_hint" .RemoteVersion AppVer) | Str2html}}</p>
66
</div>
77
{{end}}
8-
<h4 class="ui top attached header">
9-
{{.locale.Tr "admin.dashboard.statistic"}}
10-
</h4>
11-
<div class="ui attached segment">
12-
<p>
13-
{{.locale.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}
14-
</p>
15-
</div>
168
<h4 class="ui top attached header">
179
{{.locale.Tr "admin.dashboard.operations"}}
1810
</h4>

‎templates/admin/navbar.tmpl

+3
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@
5353
<div class="item">
5454
{{.locale.Tr "admin.monitor"}}
5555
<div class="menu">
56+
<a class="{{if .PageIsAdminMonitorStats}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/stats">
57+
{{.locale.Tr "admin.monitor.stats"}}
58+
</a>
5659
<a class="{{if .PageIsAdminMonitorCron}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/cron">
5760
{{.locale.Tr "admin.monitor.cron"}}
5861
</a>

‎templates/admin/stats.tmpl

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
2+
<div class="admin-setting-content">
3+
<h4 class="ui top attached header">
4+
{{.locale.Tr "admin.dashboard.statistic"}}
5+
</h4>
6+
<div class="ui attached table segment">
7+
<table class="ui very basic striped table unstackable">
8+
{{range $statsKey := .StatsKeys}}
9+
<tr>
10+
<td width="200">{{$statsKey}}</td>
11+
<td>{{index $.StatsCounter $statsKey}}</td>
12+
</tr>
13+
{{end}}
14+
</table>
15+
</div>
16+
</div>
17+
{{template "admin/layout_footer" .}}

0 commit comments

Comments
 (0)