Skip to content

Commit 9b255d5

Browse files
Rustin170506Rustin170506
authored
statistics: Do not create pseudo statistics for the auto-analysis check process (pingcap#51479)
ref pingcap#50132
1 parent ad28e23 commit 9b255d5

File tree

6 files changed

+625
-120
lines changed

6 files changed

+625
-120
lines changed

‎pkg/statistics/handle/autoanalyze/priorityqueue/BUILD.bazel

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ go_test(
3232
"queue_test.go",
3333
],
3434
flaky = True,
35-
shard_count = 17,
35+
shard_count = 22,
3636
deps = [
3737
":priorityqueue",
3838
"//pkg/parser/model",

‎pkg/statistics/handle/autoanalyze/priorityqueue/job.go

+161-15
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ import (
3131
type analyzeType string
3232

3333
const (
34-
analyzeTable analyzeType = "analyzeTable"
35-
analyzeIndex analyzeType = "analyzeIndex"
36-
analyzePartition analyzeType = "analyzePartition"
37-
analyzePartitionIndex analyzeType = "analyzePartitionIndex"
34+
analyzeTable analyzeType = "analyzeTable"
35+
analyzeIndex analyzeType = "analyzeIndex"
36+
analyzeStaticPartition analyzeType = "analyzeStaticPartition"
37+
analyzeStaticPartitionIndex analyzeType = "analyzeStaticPartitionIndex"
38+
analyzePartition analyzeType = "analyzePartition"
39+
analyzePartitionIndex analyzeType = "analyzePartitionIndex"
3840
)
3941

4042
// defaultFailedAnalysisWaitTime is the default wait time for the next analysis after a failed analysis.
@@ -50,22 +52,103 @@ type TableAnalysisJob struct {
5052
// and we don't want to analyze the same partition multiple times.
5153
// For example, the user may analyze some partitions manually, and we don't want to analyze them again.
5254
PartitionIndexes map[string][]string
53-
TableSchema string
54-
TableName string
55-
// Only set when table's indexes need to be analyzed.
55+
56+
TableSchema string
57+
// For physical or global tables.
58+
TableName string
59+
// For static pruned tables.
60+
StaticPartitionName string
61+
// Only set when table or static partition's indexes need to be analyzed.
5662
// This is only for newly added indexes.
5763
Indexes []string
64+
5865
// Only set when table's partitions need to be analyzed.
5966
// This will analyze all indexes and columns of the specified partitions.
60-
Partitions []string
61-
TableID int64
67+
Partitions []string
68+
// The global table ID.
69+
TableID int64
70+
// The static partition ID.
71+
StaticPartitionID int64
72+
6273
TableStatsVer int
6374
ChangePercentage float64
6475
TableSize float64
6576
LastAnalysisDuration time.Duration
6677
Weight float64
6778
}
6879

80+
// NewStaticPartitionTableAnalysisJob creates a new TableAnalysisJob for analyzing the static partition.
81+
func NewStaticPartitionTableAnalysisJob(
82+
schema, globalTableName string,
83+
globalTableID int64,
84+
partitionName string,
85+
partitionID int64,
86+
indexes []string,
87+
tableStatsVer int,
88+
changePercentage float64,
89+
tableSize float64,
90+
lastAnalysisDuration time.Duration,
91+
) *TableAnalysisJob {
92+
return &TableAnalysisJob{
93+
TableSchema: schema,
94+
TableName: globalTableName,
95+
TableID: globalTableID,
96+
StaticPartitionName: partitionName,
97+
StaticPartitionID: partitionID,
98+
Indexes: indexes,
99+
TableStatsVer: tableStatsVer,
100+
ChangePercentage: changePercentage,
101+
TableSize: tableSize,
102+
LastAnalysisDuration: lastAnalysisDuration,
103+
}
104+
}
105+
106+
// NewNonPartitionedTableAnalysisJob creates a new TableAnalysisJob for analyzing the table or partition.
107+
func NewNonPartitionedTableAnalysisJob(
108+
schema, tableName string,
109+
tableID int64,
110+
indexes []string,
111+
tableStatsVer int,
112+
changePercentage float64,
113+
tableSize float64,
114+
lastAnalysisDuration time.Duration,
115+
) *TableAnalysisJob {
116+
return &TableAnalysisJob{
117+
TableSchema: schema,
118+
TableName: tableName,
119+
TableID: tableID,
120+
Indexes: indexes,
121+
TableStatsVer: tableStatsVer,
122+
ChangePercentage: changePercentage,
123+
TableSize: tableSize,
124+
LastAnalysisDuration: lastAnalysisDuration,
125+
}
126+
}
127+
128+
// NewDynamicPartitionTableAnalysisJob creates a new TableAnalysisJob for analyzing dynamic partitioned table.
129+
func NewDynamicPartitionTableAnalysisJob(
130+
schema, tableName string,
131+
tableID int64,
132+
partitions []string,
133+
partitionIndexes map[string][]string,
134+
tableStatsVer int,
135+
changePercentage float64,
136+
tableSize float64,
137+
lastAnalysisDuration time.Duration,
138+
) *TableAnalysisJob {
139+
return &TableAnalysisJob{
140+
TableSchema: schema,
141+
TableName: tableName,
142+
TableID: tableID,
143+
Partitions: partitions,
144+
PartitionIndexes: partitionIndexes,
145+
TableStatsVer: tableStatsVer,
146+
ChangePercentage: changePercentage,
147+
TableSize: tableSize,
148+
LastAnalysisDuration: lastAnalysisDuration,
149+
}
150+
}
151+
69152
// HasNewlyAddedIndex checks whether the table has newly added index.
70153
func (j *TableAnalysisJob) HasNewlyAddedIndex() bool {
71154
return len(j.PartitionIndexes) > 0 || len(j.Indexes) > 0
@@ -79,9 +162,10 @@ func (j *TableAnalysisJob) IsValidToAnalyze(
79162
sctx sessionctx.Context,
80163
) (bool, string) {
81164
// No need to analyze this table.
82-
// TODO: Usually, we should not put this kind of table into the queue.
83-
if j.Weight == 0 {
84-
return false, "weight is 0"
165+
// Usually, we should not put this kind of table into the queue.
166+
// This is just a double check.
167+
if j.Weight <= 0 {
168+
return false, fmt.Sprintf("weight is less than or equal to 0: %.4f", j.Weight)
85169
}
86170

87171
// Check whether the table or partition is valid to analyze.
@@ -97,6 +181,17 @@ func (j *TableAnalysisJob) IsValidToAnalyze(
97181
); !valid {
98182
return false, failReason
99183
}
184+
} else if j.StaticPartitionName != "" {
185+
// For static partition table we only need to check the specified static partition.
186+
partitionNames := []string{j.StaticPartitionName}
187+
if valid, failReason := isValidToAnalyze(
188+
sctx,
189+
j.TableSchema,
190+
j.TableName,
191+
partitionNames...,
192+
); !valid {
193+
return false, failReason
194+
}
100195
} else {
101196
if valid, failReason := isValidToAnalyze(
102197
sctx,
@@ -219,7 +314,11 @@ func (j *TableAnalysisJob) Analyze(
219314
case analyzePartition:
220315
j.analyzePartitions(sctx, statsHandle, sysProcTracker)
221316
case analyzePartitionIndex:
222-
j.AnalyzePartitionIndexes(sctx, statsHandle, sysProcTracker)
317+
j.analyzePartitionIndexes(sctx, statsHandle, sysProcTracker)
318+
case analyzeStaticPartition:
319+
j.analyzeStaticPartition(sctx, statsHandle, sysProcTracker)
320+
case analyzeStaticPartitionIndex:
321+
j.analyzeStaticPartitionIndexes(sctx, statsHandle, sysProcTracker)
223322
}
224323
}
225324

@@ -230,7 +329,12 @@ func (j *TableAnalysisJob) getAnalyzeType() analyzeType {
230329
case len(j.Partitions) > 0:
231330
return analyzePartition
232331
case len(j.Indexes) > 0:
332+
if j.StaticPartitionName != "" {
333+
return analyzeStaticPartitionIndex
334+
}
233335
return analyzeIndex
336+
case j.StaticPartitionName != "":
337+
return analyzeStaticPartition
234338
default:
235339
return analyzeTable
236340
}
@@ -245,6 +349,15 @@ func (j *TableAnalysisJob) analyzeTable(
245349
exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...)
246350
}
247351

352+
func (j *TableAnalysisJob) analyzeStaticPartition(
353+
sctx sessionctx.Context,
354+
statsHandle statstypes.StatsHandle,
355+
sysProcTracker sessionctx.SysProcTracker,
356+
) {
357+
sql, params := j.GenSQLForAnalyzeStaticPartition()
358+
exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...)
359+
}
360+
248361
func (j *TableAnalysisJob) analyzeIndexes(
249362
sctx sessionctx.Context,
250363
statsHandle statstypes.StatsHandle,
@@ -256,6 +369,17 @@ func (j *TableAnalysisJob) analyzeIndexes(
256369
}
257370
}
258371

372+
func (j *TableAnalysisJob) analyzeStaticPartitionIndexes(
373+
sctx sessionctx.Context,
374+
statsHandle statstypes.StatsHandle,
375+
sysProcTracker sessionctx.SysProcTracker,
376+
) {
377+
for _, index := range j.Indexes {
378+
sql, params := j.GenSQLForAnalyzeStaticPartitionIndex(index)
379+
exec.AutoAnalyze(sctx, statsHandle, sysProcTracker, j.TableStatsVer, sql, params...)
380+
}
381+
}
382+
259383
// analyzePartitions performs analysis on the specified partitions.
260384
// This function uses a batch mode for efficiency. After analyzing the partitions,
261385
// it's necessary to merge their statistics. By analyzing them in batches,
@@ -283,8 +407,8 @@ func (j *TableAnalysisJob) analyzePartitions(
283407
}
284408
}
285409

286-
// AnalyzePartitionIndexes performs analysis on the specified partition indexes.
287-
func (j *TableAnalysisJob) AnalyzePartitionIndexes(
410+
// analyzePartitionIndexes performs analysis on the specified partition indexes.
411+
func (j *TableAnalysisJob) analyzePartitionIndexes(
288412
sctx sessionctx.Context,
289413
statsHandle statstypes.StatsHandle,
290414
sysProcTracker sessionctx.SysProcTracker,
@@ -332,6 +456,14 @@ func (j *TableAnalysisJob) GenSQLForAnalyzeTable() (string, []any) {
332456
return sql, params
333457
}
334458

459+
// GenSQLForAnalyzeStaticPartition generates the SQL for analyzing the specified static partition.
460+
func (j *TableAnalysisJob) GenSQLForAnalyzeStaticPartition() (string, []any) {
461+
sql := "analyze table %n.%n partition %n"
462+
params := []any{j.TableSchema, j.TableName, j.StaticPartitionName}
463+
464+
return sql, params
465+
}
466+
335467
// GenSQLForAnalyzeIndex generates the SQL for analyzing the specified index.
336468
func (j *TableAnalysisJob) GenSQLForAnalyzeIndex(index string) (string, []any) {
337469
sql := "analyze table %n.%n index %n"
@@ -340,6 +472,14 @@ func (j *TableAnalysisJob) GenSQLForAnalyzeIndex(index string) (string, []any) {
340472
return sql, params
341473
}
342474

475+
// GenSQLForAnalyzeStaticPartitionIndex generates the SQL for analyzing the specified static partition index.
476+
func (j *TableAnalysisJob) GenSQLForAnalyzeStaticPartitionIndex(index string) (string, []any) {
477+
sql := "analyze table %n.%n partition %n index %n"
478+
params := []any{j.TableSchema, j.TableName, j.StaticPartitionName, index}
479+
480+
return sql, params
481+
}
482+
343483
func (j *TableAnalysisJob) String() string {
344484
analyzeType := j.getAnalyzeType()
345485
switch analyzeType {
@@ -355,6 +495,12 @@ func (j *TableAnalysisJob) String() string {
355495
case analyzePartitionIndex:
356496
return fmt.Sprintf(`TableAnalysisJob: {AnalyzeType: partitionIndex, PartitionIndexes: %v, Schema: %s, Table: %s, TableID: %d, TableStatsVer: %d, ChangePercentage: %.2f, Weight: %.4f}`,
357497
j.PartitionIndexes, j.TableSchema, j.TableName, j.TableID, j.TableStatsVer, j.ChangePercentage, j.Weight)
498+
case analyzeStaticPartition:
499+
return fmt.Sprintf(`TableAnalysisJob: {AnalyzeType: staticPartition, Schema: %s, Table: %s, TableID: %d, StaticPartition: %s, StaticPartitionID: %d, TableStatsVer: %d, ChangePercentage: %.2f, Weight: %.4f}`,
500+
j.TableSchema, j.TableName, j.TableID, j.StaticPartitionName, j.StaticPartitionID, j.TableStatsVer, j.ChangePercentage, j.Weight)
501+
case analyzeStaticPartitionIndex:
502+
return fmt.Sprintf(`TableAnalysisJob: {AnalyzeType: staticPartitionIndex, Indexes: %s, Schema: %s, Table: %s, TableID: %d, StaticPartition: %s, StaticPartitionID: %d, TableStatsVer: %d, ChangePercentage: %.2f, Weight: %.4f}`,
503+
strings.Join(j.Indexes, ", "), j.TableSchema, j.TableName, j.TableID, j.StaticPartitionName, j.StaticPartitionID, j.TableStatsVer, j.ChangePercentage, j.Weight)
358504
default:
359505
return "TableAnalysisJob: {AnalyzeType: unknown}"
360506
}

0 commit comments

Comments
 (0)