From 8bc002e77cf4d510b432afaaedf54902854dd197 Mon Sep 17 00:00:00 2001 From: Ming Yu <95269537+ymkzpx@users.noreply.github.com> Date: Fri, 22 Jul 2022 11:53:09 +0800 Subject: [PATCH] table partition: add telemetry for partition table (#36204) close pingcap/tidb#34949 --- executor/adapter.go | 12 ++++ executor/builder.go | 40 +++++++++++ metrics/telemetry.go | 100 +++++++++++++++++++++++++++ session/session.go | 33 ++++++++- telemetry/data.go | 6 ++ telemetry/data_feature_usage.go | 14 ++++ telemetry/data_feature_usage_test.go | 45 ++++++++++++ 7 files changed, 248 insertions(+), 2 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index 5d076c2f4b456..1caf36fddd773 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -193,6 +193,18 @@ type TelemetryInfo struct { UseNonRecursive bool UseRecursive bool UseMultiSchemaChange bool + PartitionTelemetry *PartitionTelemetryInfo +} + +// PartitionTelemetryInfo records table partition telemetry information during execution. +type PartitionTelemetryInfo struct { + UseTablePartition bool + UseTablePartitionList bool + UseTablePartitionRange bool + UseTablePartitionHash bool + UseTablePartitionRangeColumns bool + UseTablePartitionListColumns bool + TablePartitionMaxPartitionsNum uint64 } // ExecStmt implements the sqlexec.Statement interface, it builds a planner.Plan to an sqlexec.Statement. diff --git a/executor/builder.go b/executor/builder.go index bdf055d95cf17..3f4939f732f21 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -990,7 +990,47 @@ func (b *executorBuilder) buildDDL(v *plannercore.DDL) Executor { if len(v.Statement.(*ast.AlterTableStmt).Specs) > 1 && b.Ti != nil { b.Ti.UseMultiSchemaChange = true } + case *ast.CreateTableStmt: + stmt := v.Statement.(*ast.CreateTableStmt) + if stmt != nil && stmt.Partition != nil { + if strings.EqualFold(b.ctx.GetSessionVars().EnableTablePartition, "OFF") { + break + } + + s := stmt.Partition + if b.Ti.PartitionTelemetry == nil { + b.Ti.PartitionTelemetry = &PartitionTelemetryInfo{} + } + b.Ti.PartitionTelemetry.TablePartitionMaxPartitionsNum = mathutil.Max(s.Num, uint64(len(s.Definitions))) + b.Ti.PartitionTelemetry.UseTablePartition = true + + switch s.Tp { + case model.PartitionTypeRange: + if s.Sub == nil { + if len(s.ColumnNames) > 0 { + b.Ti.PartitionTelemetry.UseTablePartitionRangeColumns = true + } else { + b.Ti.PartitionTelemetry.UseTablePartitionRange = true + } + } + case model.PartitionTypeHash: + if !s.Linear && s.Sub == nil { + b.Ti.PartitionTelemetry.UseTablePartitionHash = true + } + case model.PartitionTypeList: + enable := b.ctx.GetSessionVars().EnableListTablePartition + if s.Sub == nil && enable { + if len(s.ColumnNames) > 0 { + b.Ti.PartitionTelemetry.UseTablePartitionListColumns = true + } else { + b.Ti.PartitionTelemetry.UseTablePartitionList = true + } + } + + } + } } + e := &DDLExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), stmt: v.Statement, diff --git a/metrics/telemetry.go b/metrics/telemetry.go index e67af80d57204..f44f2f793c835 100644 --- a/metrics/telemetry.go +++ b/metrics/telemetry.go @@ -15,6 +15,7 @@ package metrics import ( + "github.com/pingcap/tidb/util/mathutil" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" ) @@ -35,6 +36,55 @@ var ( Name: "multi_schema_change_usage", Help: "Counter of usage of multi-schema change", }) + TelemetryTablePartitionCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_usage", + Help: "Counter of CREATE TABLE which includes of table partitioning", + }) + TelemetryTablePartitionListCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_list_usage", + Help: "Counter of CREATE TABLE which includes LIST partitioning", + }) + TelemetryTablePartitionRangeCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_range_usage", + Help: "Counter of CREATE TABLE which includes RANGE partitioning", + }) + TelemetryTablePartitionHashCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_hash_usage", + Help: "Counter of CREATE TABLE which includes HASH partitioning", + }) + TelemetryTablePartitionRangeColumnsCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_range_columns_usage", + Help: "Counter of CREATE TABLE which includes RANGE COLUMNS partitioning", + }) + TelemetryTablePartitionListColumnsCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_list_columns_usage", + Help: "Counter of CREATE TABLE which includes LIST COLUMNS partitioning", + }) + TelemetryTablePartitionMaxPartitionsCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "telemetry", + Name: "table_partition_max_partition_usage", + Help: "Counter of partitions created by CREATE TABLE statements", + }) ) // readCounter reads the value of a prometheus.Counter. @@ -94,6 +144,56 @@ func GetMultiSchemaCounter() MultiSchemaChangeUsageCounter { } } +// TablePartitionUsageCounter records the usages of table partition. +type TablePartitionUsageCounter struct { + TablePartitionCnt int64 `json:"table_partition_cnt"` + TablePartitionListCnt int64 `json:"table_partition_list_cnt"` + TablePartitionRangeCnt int64 `json:"table_partition_range_cnt"` + TablePartitionHashCnt int64 `json:"table_partition_hash_cnt"` + TablePartitionRangeColumnsCnt int64 `json:"table_partition_range_columns_cnt"` + TablePartitionListColumnsCnt int64 `json:"table_partition_list_columns_cnt"` + TablePartitionMaxPartitionsCnt int64 `json:"table_partition_max_partitions_cnt"` +} + +// Cal returns the difference of two counters. +func (c TablePartitionUsageCounter) Cal(rhs TablePartitionUsageCounter) TablePartitionUsageCounter { + return TablePartitionUsageCounter{ + TablePartitionCnt: c.TablePartitionCnt - rhs.TablePartitionCnt, + TablePartitionListCnt: c.TablePartitionListCnt - rhs.TablePartitionListCnt, + TablePartitionRangeCnt: c.TablePartitionRangeCnt - rhs.TablePartitionRangeCnt, + TablePartitionHashCnt: c.TablePartitionHashCnt - rhs.TablePartitionHashCnt, + TablePartitionRangeColumnsCnt: c.TablePartitionRangeColumnsCnt - rhs.TablePartitionRangeColumnsCnt, + TablePartitionListColumnsCnt: c.TablePartitionListColumnsCnt - rhs.TablePartitionListColumnsCnt, + TablePartitionMaxPartitionsCnt: mathutil.Max(c.TablePartitionMaxPartitionsCnt-rhs.TablePartitionMaxPartitionsCnt, rhs.TablePartitionMaxPartitionsCnt), + } +} + +// ResetTablePartitionCounter gets the TxnCommitCounter. +func ResetTablePartitionCounter(pre TablePartitionUsageCounter) TablePartitionUsageCounter { + return TablePartitionUsageCounter{ + TablePartitionCnt: readCounter(TelemetryTablePartitionCnt), + TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt), + TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt), + TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt), + TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt), + TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt), + TablePartitionMaxPartitionsCnt: mathutil.Max(readCounter(TelemetryTablePartitionMaxPartitionsCnt)-pre.TablePartitionMaxPartitionsCnt, pre.TablePartitionMaxPartitionsCnt), + } +} + +// GetTablePartitionCounter gets the TxnCommitCounter. +func GetTablePartitionCounter() TablePartitionUsageCounter { + return TablePartitionUsageCounter{ + TablePartitionCnt: readCounter(TelemetryTablePartitionCnt), + TablePartitionListCnt: readCounter(TelemetryTablePartitionListCnt), + TablePartitionRangeCnt: readCounter(TelemetryTablePartitionRangeCnt), + TablePartitionHashCnt: readCounter(TelemetryTablePartitionHashCnt), + TablePartitionRangeColumnsCnt: readCounter(TelemetryTablePartitionRangeColumnsCnt), + TablePartitionListColumnsCnt: readCounter(TelemetryTablePartitionListColumnsCnt), + TablePartitionMaxPartitionsCnt: readCounter(TelemetryTablePartitionMaxPartitionsCnt), + } +} + // NonTransactionalStmtCounter records the usages of non-transactional statements. type NonTransactionalStmtCounter struct { DeleteCount int64 `json:"delete"` diff --git a/session/session.go b/session/session.go index e4e1522390865..1a830ec6a5508 100644 --- a/session/session.go +++ b/session/session.go @@ -122,8 +122,15 @@ var ( sessionExecuteParseDurationInternal = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblInternal) sessionExecuteParseDurationGeneral = metrics.SessionExecuteParseDuration.WithLabelValues(metrics.LblGeneral) - telemetryCTEUsage = metrics.TelemetrySQLCTECnt - telemetryMultiSchemaChangeUsage = metrics.TelemetryMultiSchemaChangeCnt + telemetryCTEUsage = metrics.TelemetrySQLCTECnt + telemetryMultiSchemaChangeUsage = metrics.TelemetryMultiSchemaChangeCnt + telemetryTablePartitionUsage = metrics.TelemetryTablePartitionCnt + telemetryTablePartitionListUsage = metrics.TelemetryTablePartitionListCnt + telemetryTablePartitionRangeUsage = metrics.TelemetryTablePartitionRangeCnt + telemetryTablePartitionHashUsage = metrics.TelemetryTablePartitionHashCnt + telemetryTablePartitionRangeColumnsUsage = metrics.TelemetryTablePartitionRangeColumnsCnt + telemetryTablePartitionListColumnsUsage = metrics.TelemetryTablePartitionListColumnsCnt + telemetryTablePartitionMaxPartitionsUsage = metrics.TelemetryTablePartitionMaxPartitionsCnt ) // Session context, it is consistent with the lifecycle of a client connection. @@ -3417,6 +3424,28 @@ func (s *session) updateTelemetryMetric(es *executor.ExecStmt) { if ti.UseMultiSchemaChange { telemetryMultiSchemaChangeUsage.Inc() } + + if ti.PartitionTelemetry != nil { + if ti.PartitionTelemetry.UseTablePartition { + telemetryTablePartitionUsage.Inc() + telemetryTablePartitionMaxPartitionsUsage.Add(float64(ti.PartitionTelemetry.TablePartitionMaxPartitionsNum)) + } + if ti.PartitionTelemetry.UseTablePartitionList { + telemetryTablePartitionListUsage.Inc() + } + if ti.PartitionTelemetry.UseTablePartitionRange { + telemetryTablePartitionRangeUsage.Inc() + } + if ti.PartitionTelemetry.UseTablePartitionHash { + telemetryTablePartitionHashUsage.Inc() + } + if ti.PartitionTelemetry.UseTablePartitionRangeColumns { + telemetryTablePartitionRangeColumnsUsage.Inc() + } + if ti.PartitionTelemetry.UseTablePartitionListColumns { + telemetryTablePartitionListColumnsUsage.Inc() + } + } } // GetBuiltinFunctionUsage returns the replica of counting of builtin function usage diff --git a/telemetry/data.go b/telemetry/data.go index 119293560b631..0e357a897f356 100644 --- a/telemetry/data.go +++ b/telemetry/data.go @@ -61,7 +61,13 @@ func postReportTelemetryData() { postReportTxnUsage() postReportCTEUsage() postReportMultiSchemaChangeUsage() + postReportTablePartitionUsage() postReportSlowQueryStats() postReportNonTransactionalCounter() PostSavepointCount() } + +// PostReportTelemetryDataForTest is for test. +func PostReportTelemetryDataForTest() { + postReportTablePartitionUsage() +} diff --git a/telemetry/data_feature_usage.go b/telemetry/data_feature_usage.go index 33f8b7e4a4bc2..d0a520907513f 100644 --- a/telemetry/data_feature_usage.go +++ b/telemetry/data_feature_usage.go @@ -48,6 +48,7 @@ type featureUsage struct { NonTransactionalUsage *m.NonTransactionalStmtCounter `json:"nonTransactional"` GlobalKill bool `json:"globalKill"` MultiSchemaChange *m.MultiSchemaChangeUsageCounter `json:"multiSchemaChange"` + TablePartition *m.TablePartitionUsageCounter `json:"tablePartition"` TiFlashModeStatistics TiFlashModeStatistics `json:"TiFlashModeStatistics"` LogBackup bool `json:"logBackup"` EnablePaging bool `json:"enablePaging"` @@ -78,6 +79,8 @@ func getFeatureUsage(ctx context.Context, sctx sessionctx.Context) (*featureUsag usage.MultiSchemaChange = getMultiSchemaChangeUsageInfo() + usage.TablePartition = getTablePartitionUsageInfo() + usage.AutoCapture = getAutoCaptureUsageInfo(sctx) collectFeatureUsageFromInfoschema(sctx, &usage) @@ -219,6 +222,7 @@ var initialTxnCommitCounter metrics.TxnCommitCounter var initialCTECounter m.CTEUsageCounter var initialNonTransactionalCounter m.NonTransactionalStmtCounter var initialMultiSchemaChangeCounter m.MultiSchemaChangeUsageCounter +var initialTablePartitionCounter m.TablePartitionUsageCounter var initialSavepointStmtCounter int64 // getTxnUsageInfo gets the usage info of transaction related features. It's exported for tests. @@ -280,6 +284,16 @@ func getMultiSchemaChangeUsageInfo() *m.MultiSchemaChangeUsageCounter { return &diff } +func postReportTablePartitionUsage() { + initialTablePartitionCounter = m.ResetTablePartitionCounter(initialTablePartitionCounter) +} + +func getTablePartitionUsageInfo() *m.TablePartitionUsageCounter { + curr := m.GetTablePartitionCounter() + diff := curr.Cal(initialTablePartitionCounter) + return &diff +} + // getAutoCaptureUsageInfo gets the 'Auto Capture' usage func getAutoCaptureUsageInfo(ctx sessionctx.Context) bool { if val, err := variable.GetGlobalSystemVar(ctx.GetSessionVars(), variable.TiDBCapturePlanBaseline); err == nil { diff --git a/telemetry/data_feature_usage_test.go b/telemetry/data_feature_usage_test.go index c71e72e2c834a..7924d3bc9eb38 100644 --- a/telemetry/data_feature_usage_test.go +++ b/telemetry/data_feature_usage_test.go @@ -171,6 +171,51 @@ func TestMultiSchemaChange(t *testing.T) { require.Equal(t, int64(2), usage.MultiSchemaChange.MultiSchemaChangeUsed) } +func TestTablePartition(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + usage, err := telemetry.GetFeatureUsage(tk.Session()) + require.NoError(t, err) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionListCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionMaxPartitionsCnt) + + tk.MustExec("drop table if exists pt") + tk.MustExec("create table pt (a int,b int) partition by hash(a) partitions 4") + + usage, err = telemetry.GetFeatureUsage(tk.Session()) + require.NoError(t, err) + require.Equal(t, int64(1), usage.TablePartition.TablePartitionCnt) + require.Equal(t, int64(1), usage.TablePartition.TablePartitionHashCnt) + require.Equal(t, int64(4), usage.TablePartition.TablePartitionMaxPartitionsCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionListCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeColumnsCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionListColumnsCnt) + + telemetry.PostReportTelemetryDataForTest() + tk.MustExec("drop table if exists pt1") + tk.MustExec("create table pt1 (a int,b int) partition by range(a) (" + + "partition p0 values less than (3)," + + "partition p1 values less than (6), " + + "partition p2 values less than (9)," + + "partition p3 values less than (12)," + + "partition p4 values less than (15))") + usage, err = telemetry.GetFeatureUsage(tk.Session()) + require.NoError(t, err) + require.Equal(t, int64(1), usage.TablePartition.TablePartitionCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionHashCnt) + require.Equal(t, int64(5), usage.TablePartition.TablePartitionMaxPartitionsCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionListCnt) + require.Equal(t, int64(1), usage.TablePartition.TablePartitionRangeCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionRangeColumnsCnt) + require.Equal(t, int64(0), usage.TablePartition.TablePartitionListColumnsCnt) +} + func TestPlacementPolicies(t *testing.T) { store, clean := testkit.CreateMockStore(t) defer clean()