stmtsummary: add request-units info in statements_summary (pingcap#49504

) (pingcap#50879) ref pingcap#49318
ti-chi-bot · Feb 19, 2024 · def88be · def88be
1 parent bf23a3f
commit def88be
Show file tree

Hide file tree

Showing 14 changed files with 333 additions and 114 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -7028,13 +7028,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "013685f8bba022b46ff651baad20811d77fdf8d3c625d36e5860e74f9efb6e89",
-        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240218024828-b13f4be4e6f9",
+        sha256 = "cb66b9364d9fb4f0bc2b3dc7e2f797cec6dc931399ac3b71a50cfa4c9b8ddb06",
+        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240219022709-fa310f89d497",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240218024828-b13f4be4e6f9.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240218024828-b13f4be4e6f9.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240218024828-b13f4be4e6f9.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240218024828-b13f4be4e6f9.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240219022709-fa310f89d497.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240219022709-fa310f89d497.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240219022709-fa310f89d497.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240219022709-fa310f89d497.zip",
         ],
     )
     go_repository(

diff --git a/go.mod b/go.mod
@@ -104,7 +104,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/tdakkota/asciicheck v0.2.0
 	github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
-	github.com/tikv/client-go/v2 v2.0.8-0.20240218024828-b13f4be4e6f9
+	github.com/tikv/client-go/v2 v2.0.8-0.20240219022709-fa310f89d497
 	github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9
 	github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966
 	github.com/twmb/murmur3 v1.1.6

diff --git a/go.sum b/go.sum
@@ -995,8 +995,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
-github.com/tikv/client-go/v2 v2.0.8-0.20240218024828-b13f4be4e6f9 h1:6HtoVi0L3bFqv0fMrOf2HcpyJ73To6hi/BLPwsaXHUk=
-github.com/tikv/client-go/v2 v2.0.8-0.20240218024828-b13f4be4e6f9/go.mod h1:37p0ryKaieJbBpVDWnaPi2ZS6UFqkgpsemBLkGX2FvM=
+github.com/tikv/client-go/v2 v2.0.8-0.20240219022709-fa310f89d497 h1:dNMEvrunkV2yuFkf7YBwPPRjw9x3kPD3gl15Wa8rT4M=
+github.com/tikv/client-go/v2 v2.0.8-0.20240219022709-fa310f89d497/go.mod h1:37p0ryKaieJbBpVDWnaPi2ZS6UFqkgpsemBLkGX2FvM=
 github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9 h1:tNpO/lF/kAzl3g7JQScGi+iWSWKtPPr/YXhxYNPWsHM=
 github.com/tikv/pd/client v0.0.0-20240210135946-3488a653ddd9/go.mod h1:AwjTSpM7CgAynYwB6qTG5R5fVC9/eXlQXiTO6zDL1HI=
 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M=

diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go
@@ -1882,6 +1882,10 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
 	if tikvExecDetailRaw != nil {
 		tikvExecDetail = *(tikvExecDetailRaw.(*util.ExecDetails))
 	}
+	var ruDetail *util.RUDetails
+	if ruDetailRaw := a.GoCtx.Value(util.RUDetailsCtxKey); ruDetailRaw != nil {
+		ruDetail = ruDetailRaw.(*util.RUDetails)
+	}
 
 	if stmtCtx.WaitLockLeaseTime > 0 {
 		if execDetail.BackoffSleep == nil {
@@ -1937,6 +1941,8 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
 		Prepared:            a.isPreparedStmt,
 		KeyspaceName:        keyspaceName,
 		KeyspaceID:          keyspaceID,
+		RUDetail:            ruDetail,
+		ResourceGroupName:   sessVars.StmtCtx.ResourceGroupName,
 	}
 	if a.retryCount > 0 {
 		stmtExecInfo.ExecRetryTime = costTime - sessVars.DurationParse - sessVars.DurationCompile - time.Since(a.retryStartTime)

diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go
@@ -1362,6 +1362,13 @@ var tableStatementsSummaryCols = []columnInfo{
 	{name: stmtsummary.Charset, tp: mysql.TypeVarchar, size: 64, comment: "Sampled charset"},
 	{name: stmtsummary.Collation, tp: mysql.TypeVarchar, size: 64, comment: "Sampled collation"},
 	{name: stmtsummary.PlanHint, tp: mysql.TypeVarchar, size: 64, comment: "Sampled plan hint"},
+	{name: stmtsummary.MaxRequestUnitReadStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Max read request-unit cost of these statements"},
+	{name: stmtsummary.AvgRequestUnitReadStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Average read request-unit cost of these statements"},
+	{name: stmtsummary.MaxRequestUnitWriteStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Max write request-unit cost of these statements"},
+	{name: stmtsummary.AvgRequestUnitWriteStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Average write request-unit cost of these statements"},
+	{name: stmtsummary.MaxQueuedRcTimeStr, tp: mysql.TypeLonglong, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time of waiting for available request-units"},
+	{name: stmtsummary.AvgQueuedRcTimeStr, tp: mysql.TypeLonglong, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time of waiting for available request-units"},
+	{name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"},
 }
 
 var tableStorageStatsCols = []columnInfo{

diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go
@@ -382,4 +382,8 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem
 	addTo.sumWriteSQLRespTotal += addWith.sumWriteSQLRespTotal
 
 	addTo.sumErrors += addWith.sumErrors
+
+	addTo.StmtRUSummary.Merge(&addWith.StmtRUSummary)
+	// resourceGroupName might not be inited because when it is a evicted item.
+	addTo.resourceGroupName = addWith.resourceGroupName
 }
diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go
@@ -313,6 +313,13 @@ const (
 	Charset                           = "CHARSET"
 	Collation                         = "COLLATION"
 	PlanHint                          = "PLAN_HINT"
+	AvgRequestUnitReadStr             = "AVG_REQUEST_UNIT_READ"
+	MaxRequestUnitReadStr             = "MAX_REQUEST_UNIT_READ"
+	AvgRequestUnitWriteStr            = "AVG_REQUEST_UNIT_WRITE"
+	MaxRequestUnitWriteStr            = "MAX_REQUEST_UNIT_WRITE"
+	AvgQueuedRcTimeStr                = "AVG_QUEUED_RC_TIME"
+	MaxQueuedRcTimeStr                = "MAX_QUEUED_RC_TIME"
+	ResourceGroupName                 = "RESOURCE_GROUP"
 )
 
 type columnValueFactory func(reader *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest) interface{}
@@ -632,4 +639,25 @@ var columnValueFactoryMap = map[string]columnValueFactory{
 	PlanHint: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
 		return ssElement.planHint
 	},
+	AvgRequestUnitReadStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return avgSumFloat(ssElement.SumRRU, ssElement.execCount)
+	},
+	MaxRequestUnitReadStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.MaxRRU
+	},
+	AvgRequestUnitWriteStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return avgSumFloat(ssElement.SumWRU, ssElement.execCount)
+	},
+	MaxRequestUnitWriteStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.MaxWRU
+	},
+	AvgQueuedRcTimeStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return avgInt(int64(ssElement.SumRUWaitDuration), ssElement.execCount)
+	},
+	MaxQueuedRcTimeStr: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return int64(ssElement.MaxRUWaitDuration)
+	},
+	ResourceGroupName: func(_ *stmtSummaryReader, ssElement *stmtSummaryByDigestElement, _ *stmtSummaryByDigest) interface{} {
+		return ssElement.resourceGroupName
+	},
 }
diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go
@@ -50,6 +50,8 @@ type stmtSummaryByDigestKey struct {
 	planDigest string
 	// `hash` is the hash value of this object.
 	hash []byte
+	// `resourceGroupName` is the resource group's name of this statement is bind to.
+	resourceGroupName string
 }
 
 // Hash implements SimpleLRUCache.Key.
@@ -62,6 +64,7 @@ func (key *stmtSummaryByDigestKey) Hash() []byte {
 		key.hash = append(key.hash, hack.Slice(key.schemaName)...)
 		key.hash = append(key.hash, hack.Slice(key.prevDigest)...)
 		key.hash = append(key.hash, hack.Slice(key.planDigest)...)
+		key.hash = append(key.hash, hack.Slice(key.resourceGroupName)...)
 	}
 	return key.hash
 }
@@ -211,6 +214,9 @@ type stmtSummaryByDigestElement struct {
 	// pessimistic execution retry information.
 	execRetryCount uint
 	execRetryTime  time.Duration
+	// request-units
+	resourceGroupName string
+	StmtRUSummary
 }
 
 // StmtExecInfo records execution information of each statement.
@@ -244,11 +250,13 @@ type StmtExecInfo struct {
 	ExecRetryCount      uint
 	ExecRetryTime       time.Duration
 	execdetails.StmtExecDetails
-	ResultRows      int64
-	TiKVExecDetails util.ExecDetails
-	Prepared        bool
-	KeyspaceName    string
-	KeyspaceID      uint32
+	ResultRows        int64
+	TiKVExecDetails   util.ExecDetails
+	Prepared          bool
+	KeyspaceName      string
+	KeyspaceID        uint32
+	ResourceGroupName string
+	RUDetail          *util.RUDetails
 }
 
 // newStmtSummaryByDigestMap creates an empty stmtSummaryByDigestMap.
@@ -300,10 +308,11 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) {
 	historySize := ssMap.historySize()
 
 	key := &stmtSummaryByDigestKey{
-		schemaName: sei.SchemaName,
-		digest:     sei.Digest,
-		prevDigest: sei.PrevSQLDigest,
-		planDigest: sei.PlanDigest,
+		schemaName:        sei.SchemaName,
+		digest:            sei.Digest,
+		prevDigest:        sei.PrevSQLDigest,
+		planDigest:        sei.PlanDigest,
+		resourceGroupName: sei.ResourceGroupName,
 	}
 	// Calculate hash value in advance, to reduce the time holding the lock.
 	key.Hash()
@@ -652,20 +661,21 @@ func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalS
 		// PrevSQL is already truncated to cfg.Log.QueryLogMaxLen.
 		prevSQL: sei.PrevSQL,
 		// samplePlan needs to be decoded so it can't be truncated.
-		samplePlan:       samplePlan,
-		sampleBinaryPlan: binPlan,
-		planHint:         planHint,
-		indexNames:       sei.StmtCtx.IndexNames,
-		minLatency:       sei.TotalLatency,
-		firstSeen:        sei.StartTime,
-		lastSeen:         sei.StartTime,
-		backoffTypes:     make(map[string]int),
-		authUsers:        make(map[string]struct{}),
-		planInCache:      false,
-		planCacheHits:    0,
-		planInBinding:    false,
-		prepared:         sei.Prepared,
-		minResultRows:    math.MaxInt64,
+		samplePlan:        samplePlan,
+		sampleBinaryPlan:  binPlan,
+		planHint:          planHint,
+		indexNames:        sei.StmtCtx.IndexNames,
+		minLatency:        sei.TotalLatency,
+		firstSeen:         sei.StartTime,
+		lastSeen:          sei.StartTime,
+		backoffTypes:      make(map[string]int),
+		authUsers:         make(map[string]struct{}),
+		planInCache:       false,
+		planCacheHits:     0,
+		planInBinding:     false,
+		prepared:          sei.Prepared,
+		minResultRows:     math.MaxInt64,
+		resourceGroupName: sei.ResourceGroupName,
 	}
 	ssElement.add(sei, intervalSeconds)
 	return ssElement
@@ -889,6 +899,9 @@ func (ssElement *stmtSummaryByDigestElement) add(sei *StmtExecInfo, intervalSeco
 	ssElement.sumPDTotal += time.Duration(atomic.LoadInt64(&sei.TiKVExecDetails.WaitPDRespDuration))
 	ssElement.sumBackoffTotal += time.Duration(atomic.LoadInt64(&sei.TiKVExecDetails.BackoffDuration))
 	ssElement.sumWriteSQLRespTotal += sei.StmtExecDetails.WriteSQLRespDuration
+
+	// request-units
+	ssElement.StmtRUSummary.Add(sei.RUDetail)
 }
 
 // Truncate SQL to maxSQLLength.
@@ -950,9 +963,63 @@ func avgFloat(sum int64, count int64) float64 {
 	return 0
 }
 
+func avgSumFloat(sum float64, count int64) float64 {
+	if count > 0 {
+		return sum / float64(count)
+	}
+	return 0
+}
+
 func convertEmptyToNil(str string) interface{} {
 	if str == "" {
 		return nil
 	}
 	return str
 }
+
+// StmtRUSummary is the request-units summary for each type of statements.
+type StmtRUSummary struct {
+	SumRRU            float64       `json:"sum_rru"`
+	SumWRU            float64       `json:"sum_wru"`
+	SumRUWaitDuration time.Duration `json:"sum_ru_wait_duration"`
+	MaxRRU            float64       `json:"max_rru"`
+	MaxWRU            float64       `json:"max_wru"`
+	MaxRUWaitDuration time.Duration `json:"max_ru_wait_duration"`
+}
+
+// Add add a new sample value to the ru summary record.
+func (s *StmtRUSummary) Add(info *util.RUDetails) {
+	if info != nil {
+		rru := info.RRU()
+		s.SumRRU += rru
+		if s.MaxRRU < rru {
+			s.MaxRRU = rru
+		}
+		wru := info.WRU()
+		s.SumWRU += wru
+		if s.MaxWRU < wru {
+			s.MaxWRU = wru
+		}
+		ruWaitDur := info.RUWaitDuration()
+		s.SumRUWaitDuration += ruWaitDur
+		if s.MaxRUWaitDuration < ruWaitDur {
+			s.MaxRUWaitDuration = ruWaitDur
+		}
+	}
+}
+
+// Merge merges the value of 2 ru summary records.
+func (s *StmtRUSummary) Merge(other *StmtRUSummary) {
+	s.SumRRU += other.SumRRU
+	s.SumWRU += other.SumWRU
+	s.SumRUWaitDuration += other.SumRUWaitDuration
+	if s.MaxRRU < other.MaxRRU {
+		s.MaxRRU = other.MaxRRU
+	}
+	if s.MaxWRU < other.MaxWRU {
+		s.MaxWRU = other.MaxWRU
+	}
+	if s.MaxRUWaitDuration < other.MaxRUWaitDuration {
+		s.MaxRUWaitDuration = other.MaxRUWaitDuration
+	}
+}