Skip to content

Commit

Permalink
statement,slowquery: add request unit part in list (#1639) (#1640)
Browse files Browse the repository at this point in the history
  • Loading branch information
nolouch authored Jan 11, 2024
1 parent 2a275ed commit 41f7c80
Show file tree
Hide file tree
Showing 14 changed files with 194 additions and 12 deletions.
17 changes: 14 additions & 3 deletions pkg/apiserver/slowquery/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ type Model struct {
ConnectionID string `gorm:"column:Conn_ID" json:"connection_id"`
Success int `gorm:"column:Succ" json:"success"`

Timestamp float64 `gorm:"column:timestamp" proj:"(UNIX_TIMESTAMP(Time) + 0E0)" json:"timestamp"` // finish time
QueryTime float64 `gorm:"column:Query_time" json:"query_time"` // latency
Timestamp float64 `gorm:"column:timestamp" proj:"(UNIX_TIMESTAMP(Time) + 0E0)" json:"timestamp" related:"time"` // finish time
QueryTime float64 `gorm:"column:Query_time" json:"query_time"` // latency
ParseTime float64 `gorm:"column:Parse_time" json:"parse_time"`
CompileTime float64 `gorm:"column:Compile_time" json:"compile_time"`
RewriteTime float64 `gorm:"column:Rewrite_time" json:"rewrite_time"`
Expand Down Expand Up @@ -99,16 +99,23 @@ type Model struct {
// Computed fields
BinaryPlanJSON string `json:"binary_plan_json"` // binary plan json format
BinaryPlanText string `json:"binary_plan_text"` // binary plan plain text

// Resource Control
RU float64 `gorm:"column:RU" json:"ru" proj:"(Request_unit_write + Request_unit_read)" related:"Request_unit_write,Request_unit_read"`
QueuedTime float64 `gorm:"column:Time_queued_by_rc" json:"time_queued_by_rc"`
ResourceGroup string `gorm:"column:Resource_group" json:"resource_group"`
}

type Field struct {
ColumnName string
JSONName string
Projection string
// `related` tag is used to verify a non-existent column, which is aggregated/projection from the columns represented by related.
Related []string
}

func getFieldsAndTags() (slowQueryFields []Field) {
fields := reflectutil.GetFieldsAndTags(Model{}, []string{"gorm", "proj", "json"})
fields := reflectutil.GetFieldsAndTags(Model{}, []string{"gorm", "proj", "json", "related"})

for _, f := range fields {
sqf := Field{
Expand All @@ -117,6 +124,10 @@ func getFieldsAndTags() (slowQueryFields []Field) {
Projection: f.Tags["proj"],
}

if f.Tags["related"] != "" {
sqf.Related = strings.Split(f.Tags["related"], ",")
}

slowQueryFields = append(slowQueryFields, sqf)
}

Expand Down
24 changes: 20 additions & 4 deletions pkg/apiserver/slowquery/statement_gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"strings"

"github.com/samber/lo"

"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
)

var ErrUnknownColumn = ErrNS.NewType("unknown_column")
Expand All @@ -25,9 +27,15 @@ func genSelectStmt(tableColumns []string, reqJSONColumns []string) (string, erro

// We have both TiDB 4.x and TiDB 5.x columns listed in the model. Filter out columns that do not exist in current version TiDB schema.
fields = lo.Filter(fields, func(f Field, _ int) bool {
hasProjection := f.Projection != ""
isTableColumnValid := lo.Contains(tableColumns, f.ColumnName)
return hasProjection || isTableColumnValid
var representedColumns []string
if len(f.Related) != 0 {
representedColumns = f.Related
} else {
representedColumns = []string{f.ColumnName}
}
// For compatibility with old TiDB, we need to check if the column exists in the table.
// Dependent columns of the requested field must exist in the db schema. Otherwise, the requested field will be ignored.
return utils.IsSubsetICaseInsensitive(tableColumns, representedColumns)
})

if len(fields) == 0 {
Expand All @@ -52,7 +60,15 @@ func genOrderStmt(tableColumns []string, orderBy string, isDesc bool) (string, e
} else {
// We have both TiDB 4.x and TiDB 5.x columns listed in the model. Filter out columns that do not exist in current version TiDB schema.
fields := lo.Filter(getFieldsAndTags(), func(f Field, _ int) bool {
return lo.Contains(tableColumns, f.ColumnName)
var representedColumns []string
if len(f.Related) != 0 {
representedColumns = f.Related
} else {
representedColumns = []string{f.ColumnName}
}
// For compatibility with old TiDB, we need to check if the column exists in the table.
// Dependent columns of the requested field must exist in the db schema. Otherwise, the requested field will be ignored.
return utils.IsSubsetICaseInsensitive(tableColumns, representedColumns)
})
orderField, ok := lo.Find(fields, func(f Field) bool {
return f.JSONName == orderBy
Expand Down
6 changes: 6 additions & 0 deletions pkg/apiserver/statement/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ type Model struct {
PlanCanBeBound bool `json:"plan_can_be_bound"`
BinaryPlanJSON string `json:"binary_plan_json"`
BinaryPlanText string `json:"binary_plan_text"`

// Resource Control
AggResourceGroup string `json:"resource_group" agg:"ANY_VALUE(resource_group)"`
AggAvgRU float64 `json:"avg_ru" agg:"CAST(AVG(avg_request_unit_write + avg_request_unit_read) AS DECIMAL(64, 2))" related:"avg_request_unit_write,avg_request_unit_read"`
AggMaxRU float64 `json:"max_ru" agg:"MAX(max_request_unit_write + max_request_unit_read)" related:"max_request_unit_write,max_request_unit_read"`
AggSumRU float64 `json:"sum_ru" agg:"CAST(SUM(exec_count * (avg_request_unit_write + avg_request_unit_read)) AS DECIMAL(64, 2))" related:"avg_request_unit_write,avg_request_unit_read"`
}

// tableNames example: "d1.a1,d2.a2,d1.a1,d3.a3"
Expand Down
2 changes: 1 addition & 1 deletion tests/schema/test.CLUSTER_SLOW_QUERY-schema.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
/*!40101 SET NAMES binary*/;
/*T![placement] SET PLACEMENT_CHECKS = 0*/;
CREATE TABLE `CLUSTER_SLOW_QUERY` (`INSTANCE` VARCHAR(64) DEFAULT NULL, `Warnings` VARCHAR(64) DEFAULT NULL, `BINARY_PLAN` VARCHAR(128) DEFAULT NULL,`Time` TIMESTAMP(6) NOT NULL,`Txn_start_ts` BIGINT(20) UNSIGNED DEFAULT NULL,`User` VARCHAR(64) DEFAULT NULL,`Host` VARCHAR(64) DEFAULT NULL,`Conn_ID` BIGINT(20) UNSIGNED DEFAULT NULL,`Exec_retry_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Exec_retry_time` DOUBLE DEFAULT NULL,`Query_time` DOUBLE DEFAULT NULL,`Parse_time` DOUBLE DEFAULT NULL,`Compile_time` DOUBLE DEFAULT NULL,`Rewrite_time` DOUBLE DEFAULT NULL,`Preproc_subqueries` BIGINT(20) UNSIGNED DEFAULT NULL,`Preproc_subqueries_time` DOUBLE DEFAULT NULL,`Optimize_time` DOUBLE DEFAULT NULL,`Wait_TS` DOUBLE DEFAULT NULL,`Prewrite_time` DOUBLE DEFAULT NULL,`Wait_prewrite_binlog_time` DOUBLE DEFAULT NULL,`Commit_time` DOUBLE DEFAULT NULL,`Get_commit_ts_time` DOUBLE DEFAULT NULL,`Commit_backoff_time` DOUBLE DEFAULT NULL,`Backoff_types` VARCHAR(64) DEFAULT NULL,`Resolve_lock_time` DOUBLE DEFAULT NULL,`Local_latch_wait_time` DOUBLE DEFAULT NULL,`Write_keys` BIGINT(22) DEFAULT NULL,`Write_size` BIGINT(22) DEFAULT NULL,`Prewrite_region` BIGINT(22) DEFAULT NULL,`Txn_retry` BIGINT(22) DEFAULT NULL,`Cop_time` DOUBLE DEFAULT NULL,`Process_time` DOUBLE DEFAULT NULL,`Wait_time` DOUBLE DEFAULT NULL,`Backoff_time` DOUBLE DEFAULT NULL,`LockKeys_time` DOUBLE DEFAULT NULL,`Request_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Total_keys` BIGINT(20) UNSIGNED DEFAULT NULL,`Process_keys` BIGINT(20) UNSIGNED DEFAULT NULL,`Rocksdb_delete_skipped_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Rocksdb_key_skipped_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Rocksdb_block_cache_hit_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Rocksdb_block_read_count` BIGINT(20) UNSIGNED DEFAULT NULL,`Rocksdb_block_read_byte` BIGINT(20) UNSIGNED DEFAULT NULL,`DB` VARCHAR(64) DEFAULT NULL,`Index_names` VARCHAR(100) DEFAULT NULL,`Is_internal` TINYINT(1) DEFAULT NULL,`Digest` VARCHAR(64) DEFAULT NULL,`Stats` VARCHAR(512) DEFAULT NULL,`Cop_proc_avg` DOUBLE DEFAULT NULL,`Cop_proc_p90` DOUBLE DEFAULT NULL,`Cop_proc_max` DOUBLE DEFAULT NULL,`Cop_proc_addr` VARCHAR(64) DEFAULT NULL,`Cop_wait_avg` DOUBLE DEFAULT NULL,`Cop_wait_p90` DOUBLE DEFAULT NULL,`Cop_wait_max` DOUBLE DEFAULT NULL,`Cop_wait_addr` VARCHAR(64) DEFAULT NULL,`Mem_max` BIGINT(20) DEFAULT NULL,`Disk_max` BIGINT(20) DEFAULT NULL,`KV_total` DOUBLE DEFAULT NULL,`PD_total` DOUBLE DEFAULT NULL,`Backoff_total` DOUBLE DEFAULT NULL,`Write_sql_response_total` DOUBLE DEFAULT NULL,`Result_rows` BIGINT(22) DEFAULT NULL,`Backoff_Detail` VARCHAR(4096) DEFAULT NULL,`Prepared` TINYINT(1) DEFAULT NULL,`Succ` TINYINT(1) DEFAULT NULL,`IsExplicitTxn` TINYINT(1) DEFAULT NULL,`IsWriteCacheTable` TINYINT(1) DEFAULT NULL,`Plan_from_cache` TINYINT(1) DEFAULT NULL,`Plan_from_binding` TINYINT(1) DEFAULT NULL,`Plan` LONGTEXT DEFAULT NULL,`Plan_digest` VARCHAR(128) DEFAULT NULL,`Prev_stmt` LONGTEXT DEFAULT NULL,`Query` LONGTEXT DEFAULT NULL,PRIMARY KEY(`Time`) /*T![clustered_index] CLUSTERED */) ENGINE = InnoDB DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_BIN;
CREATE TABLE `CLUSTER_SLOW_QUERY` (`INSTANCE` varchar(64) DEFAULT NULL, `Time` timestamp(6) NOT NULL, `Txn_start_ts` bigint(20) unsigned DEFAULT NULL, `User` varchar(64) DEFAULT NULL, `Host` varchar(64) DEFAULT NULL, `Conn_ID` bigint(20) unsigned DEFAULT NULL, `Session_alias` varchar(64) DEFAULT NULL, `Exec_retry_count` bigint(20) unsigned DEFAULT NULL, `Exec_retry_time` double DEFAULT NULL, `Query_time` double DEFAULT NULL, `Parse_time` double DEFAULT NULL, `Compile_time` double DEFAULT NULL, `Rewrite_time` double DEFAULT NULL, `Preproc_subqueries` bigint(20) unsigned DEFAULT NULL, `Preproc_subqueries_time` double DEFAULT NULL, `Optimize_time` double DEFAULT NULL, `Wait_TS` double DEFAULT NULL, `Prewrite_time` double DEFAULT NULL, `Wait_prewrite_binlog_time` double DEFAULT NULL, `Commit_time` double DEFAULT NULL, `Get_commit_ts_time` double DEFAULT NULL, `Commit_backoff_time` double DEFAULT NULL, `Backoff_types` varchar(64) DEFAULT NULL, `Resolve_lock_time` double DEFAULT NULL, `Local_latch_wait_time` double DEFAULT NULL, `Write_keys` bigint(22) DEFAULT NULL, `Write_size` bigint(22) DEFAULT NULL, `Prewrite_region` bigint(22) DEFAULT NULL, `Txn_retry` bigint(22) DEFAULT NULL, `Cop_time` double DEFAULT NULL, `Process_time` double DEFAULT NULL, `Wait_time` double DEFAULT NULL, `Backoff_time` double DEFAULT NULL, `LockKeys_time` double DEFAULT NULL, `Request_count` bigint(20) unsigned DEFAULT NULL, `Total_keys` bigint(20) unsigned DEFAULT NULL, `Process_keys` bigint(20) unsigned DEFAULT NULL, `Rocksdb_delete_skipped_count` bigint(20) unsigned DEFAULT NULL, `Rocksdb_key_skipped_count` bigint(20) unsigned DEFAULT NULL, `Rocksdb_block_cache_hit_count` bigint(20) unsigned DEFAULT NULL, `Rocksdb_block_read_count` bigint(20) unsigned DEFAULT NULL, `Rocksdb_block_read_byte` bigint(20) unsigned DEFAULT NULL, `DB` varchar(64) DEFAULT NULL, `Index_names` varchar(100) DEFAULT NULL, `Is_internal` tinyint(1) DEFAULT NULL, `Digest` varchar(64) DEFAULT NULL, `Stats` varchar(512) DEFAULT NULL, `Cop_proc_avg` double DEFAULT NULL, `Cop_proc_p90` double DEFAULT NULL, `Cop_proc_max` double DEFAULT NULL, `Cop_proc_addr` varchar(64) DEFAULT NULL, `Cop_wait_avg` double DEFAULT NULL, `Cop_wait_p90` double DEFAULT NULL, `Cop_wait_max` double DEFAULT NULL, `Cop_wait_addr` varchar(64) DEFAULT NULL, `Mem_max` bigint(20) DEFAULT NULL, `Disk_max` bigint(20) DEFAULT NULL, `KV_total` double DEFAULT NULL, `PD_total` double DEFAULT NULL, `Backoff_total` double DEFAULT NULL, `Write_sql_response_total` double DEFAULT NULL, `Result_rows` bigint(22) DEFAULT NULL, `Warnings` longtext DEFAULT NULL, `Backoff_Detail` varchar(4096) DEFAULT NULL, `Prepared` tinyint(1) DEFAULT NULL, `Succ` tinyint(1) DEFAULT NULL, `IsExplicitTxn` tinyint(1) DEFAULT NULL, `IsWriteCacheTable` tinyint(1) DEFAULT NULL, `Plan_from_cache` tinyint(1) DEFAULT NULL, `Plan_from_binding` tinyint(1) DEFAULT NULL, `Has_more_results` tinyint(1) DEFAULT NULL, `Resource_group` varchar(64) DEFAULT NULL, `Request_unit_read` double DEFAULT NULL, `Request_unit_write` double DEFAULT NULL, `Time_queued_by_rc` double DEFAULT NULL, `Plan` longtext DEFAULT NULL, `Plan_digest` varchar(128) DEFAULT NULL, `Binary_plan` longtext DEFAULT NULL, `Prev_stmt` longtext DEFAULT NULL, `Query` longtext DEFAULT NULL, PRIMARY KEY (`Time`) /*T![clustered_index] CLUSTERED */ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,12 @@ export interface SlowqueryModel {
* @memberof SlowqueryModel
*/
'resolve_lock_time'?: number;
/**
*
* @type {string}
* @memberof SlowqueryModel
*/
'resource_group'?: string;
/**
*
* @type {number}
Expand Down Expand Up @@ -332,6 +338,12 @@ export interface SlowqueryModel {
* @memberof SlowqueryModel
*/
'rocksdb_key_skipped_count'?: number;
/**
* Resource Control
* @type {number}
* @memberof SlowqueryModel
*/
'ru'?: number;
/**
*
* @type {string}
Expand All @@ -344,6 +356,12 @@ export interface SlowqueryModel {
* @memberof SlowqueryModel
*/
'success'?: number;
/**
*
* @type {number}
* @memberof SlowqueryModel
*/
'time_queued_by_rc'?: number;
/**
* finish time
* @type {number}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,12 @@ export interface StatementModel {
* @memberof StatementModel
*/
'avg_rocksdb_key_skipped_count'?: number;
/**
*
* @type {number}
* @memberof StatementModel
*/
'avg_ru'?: number;
/**
*
* @type {number}
Expand Down Expand Up @@ -374,6 +380,12 @@ export interface StatementModel {
* @memberof StatementModel
*/
'max_rocksdb_key_skipped_count'?: number;
/**
*
* @type {number}
* @memberof StatementModel
*/
'max_ru'?: number;
/**
*
* @type {number}
Expand Down Expand Up @@ -464,6 +476,12 @@ export interface StatementModel {
* @memberof StatementModel
*/
'related_schemas'?: string;
/**
* Resource Control
* @type {string}
* @memberof StatementModel
*/
'resource_group'?: string;
/**
*
* @type {string}
Expand Down Expand Up @@ -506,6 +524,12 @@ export interface StatementModel {
* @memberof StatementModel
*/
'sum_latency'?: number;
/**
*
* @type {number}
* @memberof StatementModel
*/
'sum_ru'?: number;
/**
*
* @type {number}
Expand Down
23 changes: 23 additions & 0 deletions ui/packages/tidb-dashboard-client/swagger/spec.json
Original file line number Diff line number Diff line change
Expand Up @@ -5115,6 +5115,9 @@
"resolve_lock_time": {
"type": "number"
},
"resource_group": {
"type": "string"
},
"rewrite_time": {
"type": "number"
},
Expand All @@ -5134,12 +5137,19 @@
"rocksdb_key_skipped_count": {
"type": "integer"
},
"ru": {
"description": "Resource Control",
"type": "number"
},
"stats": {
"type": "string"
},
"success": {
"type": "integer"
},
"time_queued_by_rc": {
"type": "number"
},
"timestamp": {
"description": "finish time",
"type": "number"
Expand Down Expand Up @@ -5371,6 +5381,9 @@
"avg_rocksdb_key_skipped_count": {
"type": "integer"
},
"avg_ru": {
"type": "number"
},
"avg_total_keys": {
"type": "integer"
},
Expand Down Expand Up @@ -5485,6 +5498,9 @@
"max_rocksdb_key_skipped_count": {
"type": "integer"
},
"max_ru": {
"type": "number"
},
"max_total_keys": {
"type": "integer"
},
Expand Down Expand Up @@ -5533,6 +5549,10 @@
"description": "Computed fields",
"type": "string"
},
"resource_group": {
"description": "Resource Control",
"type": "string"
},
"sample_user": {
"type": "string"
},
Expand All @@ -5554,6 +5574,9 @@
"sum_latency": {
"type": "integer"
},
"sum_ru": {
"type": "number"
},
"sum_warnings": {
"type": "integer"
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,14 @@ slow_query:
rocksdb_block_read_count_tooltip: Total number of blocks RocksDB read from file (RocksDB block_read_count)
rocksdb_block_read_byte: RocksDB Read Size
rocksdb_block_read_byte_tooltip: Total number of bytes RocksDB read from file (RocksDB block_read_byte)

ru: RU
ru_tooltip: request units
resource_group: Resource Group
resource_group_tooltip: The resource group that the query belongs to
time_queued_by_rc: Time Queued by RC
time_queued_by_rc_tooltip: The wait time spent in the resource queue

common:
status:
success: Success
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,14 @@ slow_query:
rocksdb_block_read_count_tooltip: RocksDB 从文件系统中读数据的次数 (block_read_count)
rocksdb_block_read_byte: RocksDB 文件系统读数据量
rocksdb_block_read_byte_tooltip: RocksDB 从文件系统中读数据的数据量 (block_read_byte)

ru: RU
ru_tooltip: 资源单位(RU)
resource_group: 资源组
resource_group_tooltip: SQL 语句所属的资源组
time_queued_by_rc: RC 等待耗时
time_queued_by_rc_tooltip: SQL 语句在资源组队列中等待的时间

common:
status:
success: 成功
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ export function slowQueryColumns(
tcf.bar.single('rocksdb_block_read_byte', 'bytes', rows).patchConfig({
minWidth: 220,
maxWidth: 250
})
}),
// resource control
tcf.bar.single('ru', 'none', rows),
tcf.textWithTooltip('resource_group', rows),
tcf.bar.single('time_queued_by_rc', 's', rows)
])
}
Original file line number Diff line number Diff line change
Expand Up @@ -184,3 +184,10 @@ statement:
avg_rocksdb_block_read_byte: Mean RocksDB FS Read Size
avg_rocksdb_block_read_byte_tooltip: Total number of bytes RocksDB read from file (RocksDB block_read_byte)
max_rocksdb_block_read_byte: Max RocksDB FS Read Size

resource_group: Resource Group
resource_group_tooltip: The resource group that the query belongs to
avg_ru: Mean RU
avg_ru_tooltip: The average number of request units (RU) consumed by the query
sum_ru: Total RU
sum_ru_tooltip: The total number of request units (RU) consumed by the query
Original file line number Diff line number Diff line change
Expand Up @@ -191,3 +191,10 @@ statement:
avg_rocksdb_block_read_byte: RocksDB 文件系统平均读数据量
avg_rocksdb_block_read_byte_tooltip: RocksDB 从文件系统中读数据的数据量 (block_read_byte)
max_rocksdb_block_read_byte: RocksDB 文件系统最大读数据量

resource_group: 资源组
resource_group_tooltip: SQL 语句所属的资源组
avg_ru: 平均 RU
avg_ru_tooltip: SQL 语句的平均 RU
sum_ru: 累积 RU
sum_ru_tooltip: SQL 语句的 RU 累积值
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ export const derivedFields = {
avg_rocksdb_block_read_byte: genDerivedBarSources(
'avg_rocksdb_block_read_byte',
'max_rocksdb_block_read_byte'
)
),
avg_ru: genDerivedBarSources('avg_ru', 'max_ru')
}

//////////////////////////////////////////
Expand Down Expand Up @@ -172,7 +173,6 @@ export function statementColumns(
showFullSQL?: boolean
): IColumn[] {
const tcf = new TableColumnFactory(TRANS_KEY_PREFIX, tableSchemaColumns)

return tcf.columns([
evictedRenderColumn(
tcf.sqlText('digest_text', showFullSQL, rows).getConfig()
Expand Down Expand Up @@ -274,7 +274,15 @@ export function statementColumns(
minWidth: 220,
maxWidth: 250
}
)
),
//resource control
tcf.textWithTooltip('resource_group', rows),
avgMaxColumn(tcf, 'avg_ru', 'none', rows),
tcf.textWithTooltip('sum_ru', rows).patchConfig({
minWidth: 100,
maxWidth: 300,
columnActionsMode: ColumnActionsMode.clickable
})
])
}

Expand Down
Loading

0 comments on commit 41f7c80

Please sign in to comment.