Skip to content

Commit 789de8d

Browse files
Rustin170506Rustin170506
authored
statistics: add better API and better logs for lock stats (#47024)
ref #46351
1 parent fdff1e6 commit 789de8d

13 files changed

+418
-95
lines changed

‎executor/lockstats/BUILD.bazel

+15-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
load("@io_bazel_rules_go//go:def.bzl", "go_library")
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
22

33
go_library(
44
name = "lockstats",
@@ -19,3 +19,17 @@ go_library(
1919
"@com_github_pingcap_errors//:errors",
2020
],
2121
)
22+
23+
go_test(
24+
name = "lockstats_test",
25+
timeout = "short",
26+
srcs = ["lock_stats_executor_test.go"],
27+
embed = [":lockstats"],
28+
flaky = True,
29+
deps = [
30+
"//infoschema",
31+
"//parser/ast",
32+
"//parser/model",
33+
"@com_github_stretchr_testify//require",
34+
],
35+
)

‎executor/lockstats/lock_stats_executor.go

+34-13
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package lockstats
1616

1717
import (
1818
"context"
19+
"fmt"
1920

2021
"github.com/pingcap/errors"
2122
"github.com/pingcap/tidb/domain"
@@ -51,12 +52,13 @@ func (e *LockExec) Next(_ context.Context, _ *chunk.Chunk) error {
5152
is := do.InfoSchema()
5253

5354
if e.onlyLockPartitions() {
54-
tableName := e.Tables[0]
55-
tid, pidNames, err := populatePartitionIDAndNames(tableName, tableName.PartitionNames, is)
55+
table := e.Tables[0]
56+
tid, pidNames, err := populatePartitionIDAndNames(table, table.PartitionNames, is)
5657
if err != nil {
5758
return err
5859
}
5960

61+
tableName := fmt.Sprintf("%s.%s", table.Schema.L, table.Name.L)
6062
msg, err := h.LockPartitions(tid, tableName, pidNames)
6163
if err != nil {
6264
return err
@@ -65,12 +67,12 @@ func (e *LockExec) Next(_ context.Context, _ *chunk.Chunk) error {
6567
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(errors.New(msg))
6668
}
6769
} else {
68-
tids, pids, err := populateTableAndPartitionIDs(e.Tables, is)
70+
tidAndNames, pidAndNames, err := populateTableAndPartitionIDs(e.Tables, is)
6971
if err != nil {
7072
return err
7173
}
7274

73-
msg, err := h.LockTables(tids, pids, e.Tables)
75+
msg, err := h.LockTables(tidAndNames, pidAndNames)
7476
if err != nil {
7577
return err
7678
}
@@ -97,15 +99,23 @@ func (*LockExec) Open(context.Context) error {
9799
}
98100

99101
// populatePartitionIDAndNames returns the table ID and partition IDs for the given table name and partition names.
100-
func populatePartitionIDAndNames(tableName *ast.TableName, partitionNames []model.CIStr, is infoschema.InfoSchema) (int64, map[int64]string, error) {
101-
tbl, err := is.TableByName(tableName.Schema, tableName.Name)
102+
func populatePartitionIDAndNames(
103+
table *ast.TableName,
104+
partitionNames []model.CIStr,
105+
is infoschema.InfoSchema,
106+
) (int64, map[int64]string, error) {
107+
if len(partitionNames) == 0 {
108+
return 0, nil, errors.New("partition list should not be empty")
109+
}
110+
tbl, err := is.TableByName(table.Schema, table.Name)
102111
if err != nil {
103112
return 0, nil, err
104113
}
105114

106115
pi := tbl.Meta().GetPartitionInfo()
107116
if pi == nil {
108-
return 0, nil, errors.Errorf("table %s is not a partition table", tableName.Name)
117+
return 0, nil, errors.Errorf("table %s is not a partition table",
118+
fmt.Sprintf("%s.%s", table.Schema.L, table.Name.L))
109119
}
110120

111121
pidNames := make(map[int64]string, len(partitionNames))
@@ -121,25 +131,36 @@ func populatePartitionIDAndNames(tableName *ast.TableName, partitionNames []mode
121131
}
122132

123133
// populateTableAndPartitionIDs returns table IDs and partition IDs for the given table names.
124-
func populateTableAndPartitionIDs(tables []*ast.TableName, is infoschema.InfoSchema) ([]int64, []int64, error) {
125-
tids := make([]int64, 0, len(tables))
126-
pids := make([]int64, 0)
134+
func populateTableAndPartitionIDs(
135+
tables []*ast.TableName,
136+
is infoschema.InfoSchema,
137+
) (map[int64]string, map[int64]string, error) {
138+
if len(tables) == 0 {
139+
return nil, nil, errors.New("table list should not be empty")
140+
}
141+
142+
tidAndNames := make(map[int64]string, len(tables))
143+
pidAndNames := make(map[int64]string, len(tables))
127144

128145
for _, table := range tables {
129146
tbl, err := is.TableByName(table.Schema, table.Name)
130147
if err != nil {
131148
return nil, nil, err
132149
}
133-
tids = append(tids, tbl.Meta().ID)
150+
tidAndNames[tbl.Meta().ID] = fmt.Sprintf("%s.%s", table.Schema.L, table.Name.L)
134151

135152
pi := tbl.Meta().GetPartitionInfo()
136153
if pi == nil {
137154
continue
138155
}
139156
for _, p := range pi.Definitions {
140-
pids = append(pids, p.ID)
157+
pidAndNames[p.ID] = genFullPartitionName(table, p.Name.L)
141158
}
142159
}
143160

144-
return tids, pids, nil
161+
return tidAndNames, pidAndNames, nil
162+
}
163+
164+
func genFullPartitionName(table *ast.TableName, partitionName string) string {
165+
return fmt.Sprintf("%s.%s partition (%s)", table.Schema.L, table.Name.L, partitionName)
145166
}

‎executor/lockstats/lock_stats_executor_test.go

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// Copyright 2023 PingCAP, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package lockstats
16+
17+
import (
18+
"testing"
19+
20+
"github.com/pingcap/tidb/infoschema"
21+
"github.com/pingcap/tidb/parser/ast"
22+
"github.com/pingcap/tidb/parser/model"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func TestPopulatePartitionIDAndNames(t *testing.T) {
27+
fakeInfo := infoschema.MockInfoSchema([]*model.TableInfo{
28+
tInfo(1, "t1", "p1", "p2"),
29+
})
30+
31+
table := &ast.TableName{
32+
Schema: model.NewCIStr("test"),
33+
Name: model.NewCIStr("t1"),
34+
PartitionNames: []model.CIStr{
35+
model.NewCIStr("p1"),
36+
model.NewCIStr("p2"),
37+
},
38+
}
39+
40+
gotTID, gotPIDNames, err := populatePartitionIDAndNames(table, table.PartitionNames, fakeInfo)
41+
require.NoError(t, err)
42+
require.Equal(t, int64(1), gotTID)
43+
require.Equal(t, map[int64]string{
44+
2: "p1",
45+
3: "p2",
46+
}, gotPIDNames)
47+
48+
// Empty partition names.
49+
_, _, err = populatePartitionIDAndNames(nil, nil, fakeInfo)
50+
require.Error(t, err)
51+
}
52+
53+
func TestPopulateTableAndPartitionIDs(t *testing.T) {
54+
fakeInfo := infoschema.MockInfoSchema([]*model.TableInfo{
55+
tInfo(1, "t1", "p1", "p2"),
56+
tInfo(4, "t2"),
57+
})
58+
59+
tables := []*ast.TableName{
60+
{
61+
Schema: model.NewCIStr("test"),
62+
Name: model.NewCIStr("t1"),
63+
PartitionNames: []model.CIStr{
64+
model.NewCIStr("p1"),
65+
model.NewCIStr("p2"),
66+
},
67+
},
68+
{
69+
Schema: model.NewCIStr("test"),
70+
Name: model.NewCIStr("t2"),
71+
},
72+
}
73+
74+
gotTIDAndNames, gotPIDAndNames, err := populateTableAndPartitionIDs(tables, fakeInfo)
75+
require.NoError(t, err)
76+
require.Equal(t, map[int64]string{
77+
1: "test.t1",
78+
4: "test.t2",
79+
}, gotTIDAndNames)
80+
require.Equal(t, map[int64]string{
81+
2: "test.t1 partition (p1)",
82+
3: "test.t1 partition (p2)",
83+
}, gotPIDAndNames)
84+
85+
// Empty table list.
86+
_, _, err = populateTableAndPartitionIDs(nil, fakeInfo)
87+
require.Error(t, err)
88+
}
89+
90+
func tInfo(id int, tableName string, partitionNames ...string) *model.TableInfo {
91+
tbl := &model.TableInfo{
92+
ID: int64(id),
93+
Name: model.NewCIStr(tableName),
94+
}
95+
if len(partitionNames) > 0 {
96+
tbl.Partition = &model.PartitionInfo{
97+
Enable: true,
98+
}
99+
for i, partitionName := range partitionNames {
100+
tbl.Partition.Definitions = append(tbl.Partition.Definitions, model.PartitionDefinition{
101+
ID: int64(id + 1 + i),
102+
Name: model.NewCIStr(partitionName),
103+
})
104+
}
105+
}
106+
107+
return tbl
108+
}

‎executor/lockstats/unlock_stats_executor.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package lockstats
1616

1717
import (
1818
"context"
19+
"fmt"
1920

2021
"github.com/pingcap/errors"
2122
"github.com/pingcap/tidb/domain"
@@ -48,11 +49,12 @@ func (e *UnlockExec) Next(context.Context, *chunk.Chunk) error {
4849
is := do.InfoSchema()
4950

5051
if e.onlyUnlockPartitions() {
51-
tableName := e.Tables[0]
52-
tid, pidNames, err := populatePartitionIDAndNames(tableName, tableName.PartitionNames, is)
52+
table := e.Tables[0]
53+
tid, pidNames, err := populatePartitionIDAndNames(table, table.PartitionNames, is)
5354
if err != nil {
5455
return err
5556
}
57+
tableName := fmt.Sprintf("%s.%s", table.Schema.O, table.Name.O)
5658
msg, err := h.RemoveLockedPartitions(tid, tableName, pidNames)
5759
if err != nil {
5860
return err
@@ -61,11 +63,11 @@ func (e *UnlockExec) Next(context.Context, *chunk.Chunk) error {
6163
e.Ctx().GetSessionVars().StmtCtx.AppendWarning(errors.New(msg))
6264
}
6365
} else {
64-
tids, pids, err := populateTableAndPartitionIDs(e.Tables, is)
66+
tidAndNames, pidAndNames, err := populateTableAndPartitionIDs(e.Tables, is)
6567
if err != nil {
6668
return err
6769
}
68-
msg, err := h.RemoveLockedTables(tids, pids, e.Tables)
70+
msg, err := h.RemoveLockedTables(tidAndNames, pidAndNames)
6971
if err != nil {
7072
return err
7173
}

‎statistics/handle/handletest/lockstats/BUILD.bazel

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ go_test(
99
"main_test.go",
1010
],
1111
flaky = True,
12-
shard_count = 12,
12+
shard_count = 14,
1313
deps = [
1414
"//config",
1515
"//domain",

‎statistics/handle/handletest/lockstats/lock_partition_stats_test.go

+24-2
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func TestLockAndUnlockPartitionStatsRepeatedly(t *testing.T) {
148148
// Lock the partition again and check the warning.
149149
tk.MustExec("lock stats t partition p0")
150150
tk.MustQuery("show warnings").Check(testkit.Rows(
151-
"Warning 1105 skip locking locked table: test.t partition (p0)",
151+
"Warning 1105 skip locking locked partition of table test.t: p0",
152152
))
153153

154154
// Unlock the partition.
@@ -160,7 +160,7 @@ func TestLockAndUnlockPartitionStatsRepeatedly(t *testing.T) {
160160
// Unlock the partition again and check the warning.
161161
tk.MustExec("unlock stats t partition p0")
162162
tk.MustQuery("show warnings").Check(testkit.Rows(
163-
"Warning 1105 skip unlocking unlocked table: test.t partition (p0)",
163+
"Warning 1105 skip unlocking unlocked partition of table test.t: p0",
164164
))
165165
}
166166

@@ -260,6 +260,28 @@ func TestUnlockTheWholeTableWouldUnlockLockedPartitionsAndGenerateWarning(t *tes
260260
require.Equal(t, 0, num)
261261
}
262262

263+
func TestSkipLockALotOfPartitions(t *testing.T) {
264+
store, _ := testkit.CreateMockStoreAndDomain(t)
265+
tk := testkit.NewTestKit(t, store)
266+
tk.MustExec("set @@tidb_analyze_version = 1")
267+
tk.MustExec("use test")
268+
tk.MustExec("drop table if exists t")
269+
tk.MustExec("create table t(a int, b varchar(10), index idx_b (b)) partition by range(a) " +
270+
"(partition p0 values less than (10), partition p1 values less than (20), " +
271+
"partition a values less than (30), " +
272+
"partition b values less than (40), " +
273+
"partition g values less than (90), " +
274+
"partition h values less than (100))")
275+
276+
tk.MustExec("lock stats t partition p0, p1, a, b, g, h")
277+
278+
// Skip locking a lot of partitions.
279+
tk.MustExec("lock stats t partition p0, p1, a, b, g, h")
280+
tk.MustQuery("show warnings").Check(testkit.Rows(
281+
"Warning 1105 skip locking locked partitions of table test.t: a, b, g, h, p0, p1",
282+
))
283+
}
284+
263285
func setupTestEnvironmentWithPartitionedTableT(t *testing.T) (kv.Storage, *testkit.TestKit, *model.TableInfo) {
264286
store, dom := testkit.CreateMockStoreAndDomain(t)
265287
tk := testkit.NewTestKit(t, store)

‎statistics/handle/handletest/lockstats/lock_table_stats_test.go

+20
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,26 @@ func TestShowStatsLockedTablePrivilege(t *testing.T) {
316316
require.Len(t, rows, 1)
317317
}
318318

319+
func TestSkipLockALotOfTables(t *testing.T) {
320+
store, _ := testkit.CreateMockStoreAndDomain(t)
321+
tk := testkit.NewTestKit(t, store)
322+
tk.MustExec("set @@tidb_analyze_version = 1")
323+
tk.MustExec("use test")
324+
tk.MustExec("drop table if exists t; drop table if exists a; drop table if exists x; drop table if exists y; drop table if exists z")
325+
tk.MustExec("create table t(a int, b varchar(10), index idx_b (b)); " +
326+
"create table a(a int, b varchar(10), index idx_b (b)); " +
327+
"create table x(a int, b varchar(10), index idx_b (b)); " +
328+
"create table y(a int, b varchar(10), index idx_b (b)); " +
329+
"create table z(a int, b varchar(10), index idx_b (b))")
330+
tk.MustExec("lock stats test.t, test.a, test.x, test.y, test.z")
331+
332+
// Skip locking a lot of tables.
333+
tk.MustExec("lock stats test.t, test.a, test.x, test.y, test.z")
334+
tk.MustQuery("show warnings").Check(testkit.Rows(
335+
"Warning 1105 skip locking locked tables: test.a, test.t, test.x, test.y, test.z",
336+
))
337+
}
338+
319339
func setupTestEnvironmentWithTableT(t *testing.T) (kv.Storage, *testkit.TestKit, *model.TableInfo) {
320340
store, dom := testkit.CreateMockStoreAndDomain(t)
321341
tk := testkit.NewTestKit(t, store)

0 commit comments

Comments
 (0)