inforschema, executor, util/kvcache, util/statement_summary : Add STATEMENTS_SUMMARY_EVICTED into information_schema

executor/builder.go

@@ -1526,6 +1526,7 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo
 			strings.ToLower(infoschema.TableTiKVStoreStatus),
 			strings.ToLower(infoschema.TableStatementsSummary),
 			strings.ToLower(infoschema.TableStatementsSummaryHistory),
+			strings.ToLower(infoschema.TableStatementsSummaryEvicted),
 			strings.ToLower(infoschema.ClusterTableStatementsSummary),
 			strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory),
 			strings.ToLower(infoschema.TablePlacementPolicy),

executor/infoschema_reader.go

@@ -149,6 +149,8 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
 			infoschema.TableClientErrorsSummaryByUser,
 			infoschema.TableClientErrorsSummaryByHost:
 			err = e.setDataForClientErrorsSummary(sctx, e.table.Name.O)
+		case infoschema.TableStatementsSummaryEvicted:
+			err = e.setDataForStmtSummaryEvicted(sctx, e.table.Name.O)
 		}
 		if err != nil {
 			return nil, err
@@ -2011,6 +2013,14 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context
 	return nil
 }
 
+func (e *memtableRetriever) setDataForStmtSummaryEvicted(ctx sessionctx.Context, tableName string) error {
+	switch tableName {
+	case infoschema.TableStatementsSummaryEvicted:
+		e.rows = stmtsummary.StmtSummaryByDigestMap.ToEvictedCountDatum()
+	}
+	return nil
+}
+
 type hugeMemTableRetriever struct {
 	dummyCloser
 	table       *model.TableInfo

infoschema/tables.go

@@ -161,6 +161,8 @@ const (
 	TableClientErrorsSummaryByUser = "CLIENT_ERRORS_SUMMARY_BY_USER"
 	// TableClientErrorsSummaryByHost is the string constant of client errors table.
 	TableClientErrorsSummaryByHost = "CLIENT_ERRORS_SUMMARY_BY_HOST"
+	// TableStatementsSummaryEvicted is the string constant of statements summary evicted table.
+	TableStatementsSummaryEvicted = "STATEMENTS_SUMMARY_EVICTED"
 )
 
 var tableIDMap = map[string]int64{
@@ -233,6 +235,7 @@ var tableIDMap = map[string]int64{
 	TableClientErrorsSummaryGlobal:          autoid.InformationSchemaDBID + 67,
 	TableClientErrorsSummaryByUser:          autoid.InformationSchemaDBID + 68,
 	TableClientErrorsSummaryByHost:          autoid.InformationSchemaDBID + 69,
+	TableStatementsSummaryEvicted:           autoid.InformationSchemaDBID + 70,
 }
 
 type columnInfo struct {
@@ -1332,6 +1335,12 @@ var tableClientErrorsSummaryByHostCols = []columnInfo{
 	{name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26},
 }
 
+var tableStatementsSummaryEvictedCols = []columnInfo{
+	{name: "BEGIN_TIME", tp: mysql.TypeTimestamp, size: 26},
+	{name: "END_TIME", tp: mysql.TypeTimestamp, size: 26},
+	{name: "EVICTED_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag},
+}
+
 // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo.
 // The returned description string may be:
 //  - "NOT_SHARDED": for tables that SHARD_ROW_ID_BITS is not specified.
@@ -1701,6 +1710,7 @@ var tableNameToColumns = map[string][]columnInfo{
 	TableClientErrorsSummaryGlobal:          tableClientErrorsSummaryGlobalCols,
 	TableClientErrorsSummaryByUser:          tableClientErrorsSummaryByUserCols,
 	TableClientErrorsSummaryByHost:          tableClientErrorsSummaryByHostCols,
+	TableStatementsSummaryEvicted:           tableStatementsSummaryEvictedCols,
 }
 
 func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) {

util/kvcache/simple_lru.go

@@ -118,6 +118,9 @@ func (l *SimpleLRUCache) Put(key Key, value Value) {
 		if l.size > l.capacity {
 			lru := l.cache.Back()
 			l.cache.Remove(lru)
+			if l.onEvict != nil {
+				l.onEvict(lru.Value.(*cacheEntry).key, lru.Value.(*cacheEntry).value)
+			}
 			delete(l.elements, string(lru.Value.(*cacheEntry).key.Hash()))
 			l.size--
 		}

util/stmtsummary/evicted.go

@@ -0,0 +1,319 @@
+package stmtsummary
+
+import (
+	"container/list"
+	"time"
+
+	"github.com/pingcap/parser/mysql"
+	"github.com/pingcap/tidb/types"
+)
+
+// stmtSummaryByDigestEvicted contents digests evicted from stmtSummaryByDigestMap
+type stmtSummaryByDigestEvicted struct {
+	// record evicted data in intervals
+	// latest history data is Back()
+	history *list.List
+}
+
+// element being stored in stmtSummaryByDigestEvicted
+type stmtSummaryByDigestEvictedElement struct {
+	// *Kinds* of digest being evicted
+	digestKeyMap map[string]struct{}
+
+	// summary of digest being evicted
+	sum *stmtSummaryByDigestElement
+}
+
+// spawn a new pointer to stmtSummaryByDigestEvicted
+func newStmtSummaryByDigestEvicted() *stmtSummaryByDigestEvicted {
+	return &stmtSummaryByDigestEvicted{
+		history: list.New(),
+	}
+}
+
+// spawn a new pointer to stmtSummaryByDigestEvictedElement
+func newStmtSummaryByDigestEvictedElement(beginTimeForCurrentInterval int64, intervalSeconds int64) *stmtSummaryByDigestEvictedElement {
+	ssElement := new(stmtSummaryByDigestElement)
+	ssElement.beginTime = beginTimeForCurrentInterval
+	ssElement.endTime = beginTimeForCurrentInterval + intervalSeconds
+	return &stmtSummaryByDigestEvictedElement{
+		digestKeyMap: make(map[string]struct{}),
+		sum:          ssElement,
+	}
+}
+
+// AddEvicted is used add an evicted record to stmtSummaryByDigestEvicted
+func (ssbde *stmtSummaryByDigestEvicted) AddEvicted(evictedKey *stmtSummaryByDigestKey, evictedValue *stmtSummaryByDigest, historySize int) {
+
+	// *need to get optimized*!!
+	evictedValue.Lock()
+	defer evictedValue.Unlock()
+	for e := evictedValue.history.Back(); e != nil; e = e.Prev() {
+		eBeginTime := e.Value.(*stmtSummaryByDigestElement).beginTime
+		eEndTime := e.Value.(*stmtSummaryByDigestElement).endTime
+
+		// prevent exceeding history size
+		for ssbde.history.Len() >= historySize && ssbde.history.Len() > 1 {
+			ssbde.history.Remove(ssbde.history.Front())
+		}
+
+		// look for match history interval
+		// no record in history
+		if ssbde.history.Len() == 0 && historySize > 0 {
+			beginTime := eBeginTime
+			intervalSeconds := eEndTime - eBeginTime
+			record := newStmtSummaryByDigestEvictedElement(beginTime, intervalSeconds)
+			record.addEvicted(evictedKey, e.Value.(*stmtSummaryByDigestElement))
+			ssbde.history.PushBack(record)
+			continue
+		}
+
+		for h := ssbde.history.Back(); h != nil; h = h.Prev() {
+			sBeginTime := h.Value.(*stmtSummaryByDigestEvictedElement).sum.beginTime
+			sEndTime := h.Value.(*stmtSummaryByDigestEvictedElement).sum.endTime
+
+			if sBeginTime <= eBeginTime &&
+				sEndTime >= eEndTime {
+				// is in this history interval
+				h.Value.(*stmtSummaryByDigestEvictedElement).addEvicted(evictedKey, e.Value.(*stmtSummaryByDigestElement))
+				break
+			}
+
+			if sEndTime <= eBeginTime {
+				// digest is young, insert into new interval after this history interval
+				beginTime := eBeginTime
+				intervalSeconds := eEndTime - eBeginTime
+				record := newStmtSummaryByDigestEvictedElement(beginTime, intervalSeconds)
+				record.addEvicted(evictedKey, e.Value.(*stmtSummaryByDigestElement))
+				ssbde.history.InsertAfter(record, h)
+				break
+			}
+
+			if sBeginTime > eEndTime {
+				// digestElement is old
+				if h != ssbde.history.Front() {
+					// check older history digestEvictedElement
+					continue
+				} else if ssbde.history.Len() >= historySize {
+					// out of history size, abandon
+					break
+				} else {
+					// is oldest digest
+					// creat a digestEvictedElement and PushFront!
+					beginTime := eBeginTime
+					intervalSeconds := eEndTime - eBeginTime
+					record := newStmtSummaryByDigestEvictedElement(beginTime, intervalSeconds)
+					record.addEvicted(evictedKey, e.Value.(*stmtSummaryByDigestElement))
+					ssbde.history.PushFront(record)
+					break
+				}
+			}
+		}
+	}
+}
+
+// Clear up all records in stmtSummaryByDigestEvicted
+func (ssbde *stmtSummaryByDigestEvicted) Clear() {
+	ssbde.history.Init()
+}
+
+// add an evicted record to stmtSummaryByDigestEvictedElement
+func (seElement *stmtSummaryByDigestEvictedElement) addEvicted(digestKey *stmtSummaryByDigestKey, digestValue *stmtSummaryByDigestElement) {
+	if digestKey != nil {
+		seElement.digestKeyMap[string(digestKey.Hash())] = struct{}{}
+	}
+	sumEvicted(seElement.sum, digestValue)
+}
+
+// ToCurrentOtherDatum converts current evicted record to `other` record's datum
+func (ssbde *stmtSummaryByDigestEvicted) ToCurrentOtherDatum() []types.Datum {
+	induceSsbd := new(stmtSummaryByDigest)
+	induceSsbd.stmtType = ""
+	induceSsbd.schemaName = ""
+	induceSsbd.digest = ""
+	induceSsbd.normalizedSQL = ""
+	induceSsbd.planDigest = ""
+	return ssbde.history.Back().Value.(*stmtSummaryByDigestEvictedElement).toOtherDatum(induceSsbd)
+}
+
+// ToHistoryOtherDatum converts history evicted record to `other` record's datum
+func (ssbde *stmtSummaryByDigestEvicted) ToHistoryOtherDatum() [][]types.Datum {
+	induceSsbd := new(stmtSummaryByDigest)
+
+	var records [][]types.Datum
+	for e := ssbde.history.Front(); e != nil; e = e.Next() {
+		if record := e.Value.(*stmtSummaryByDigestEvictedElement).toOtherDatum(induceSsbd); record != nil {
+			records = append(records, record)
+		}
+	}
+	return records
+}
+
+// ToEvictedCountDatum converts history evicted record to `evicted count` record's datum
+func (ssbde *stmtSummaryByDigestEvicted) ToEvictedCountDatum() [][]types.Datum {
+	records := make([][]types.Datum, 0, ssbde.history.Len())
+	for e := ssbde.history.Front(); e != nil; e = e.Next() {
+		if record := e.Value.(*stmtSummaryByDigestEvictedElement).toEvictedCountDatum(); record != nil {
+			records = append(records, record)
+		}
+	}
+	return records
+}
+
+// toOtherDatum converts evicted record to `other` record's datum
+func (seElement *stmtSummaryByDigestEvictedElement) toOtherDatum(ssbd *stmtSummaryByDigest) []types.Datum {
+	return seElement.sum.toDatum(ssbd)
+}
+
+// toEvictedCountDatum converts evicted record to `EvictedCount` record's datum
+func (seElement *stmtSummaryByDigestEvictedElement) toEvictedCountDatum() []types.Datum {
+	datum := types.MakeDatums(
+		types.NewTime(types.FromGoTime(time.Unix(seElement.sum.beginTime, 0)), mysql.TypeTimestamp, 0),
+		types.NewTime(types.FromGoTime(time.Unix(seElement.sum.endTime, 0)), mysql.TypeTimestamp, 0),
+		int64(len(seElement.digestKeyMap)),
+	)
+	return datum
+}
+
+func (ssMap *stmtSummaryByDigestMap) ToEvictedCountDatum() [][]types.Datum {
+	return ssMap.other.ToEvictedCountDatum()
+}
+
+// sumEvicted sum addWith into addTo
+func sumEvicted(sumTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElement) {
+	// Time duration relation: addWith ⊆ sumTo
+	if sumTo.beginTime < addWith.beginTime {
+		sumTo.beginTime = addWith.beginTime
+	}
+	if sumTo.endTime > addWith.beginTime {
+		sumTo.endTime = addWith.endTime
+	}
+	// basic
+	sumTo.execCount += addWith.execCount
+	sumTo.sumErrors += addWith.sumErrors
+	sumTo.sumWarnings += addWith.sumWarnings
+	// latency
+	sumTo.sumLatency += addWith.sumLatency
+	sumTo.sumParseLatency += addWith.sumParseLatency
+	sumTo.sumCompileLatency += addWith.sumCompileLatency
+	if sumTo.maxLatency < addWith.maxLatency {
+		sumTo.maxLatency = addWith.maxLatency
+	}
+	if sumTo.maxCompileLatency < addWith.maxCompileLatency {
+		sumTo.maxCompileLatency = addWith.maxCompileLatency
+	}
+	if sumTo.minLatency > addWith.minLatency {
+		sumTo.minLatency = addWith.minLatency
+	}
+	// coprocessor
+	sumTo.sumNumCopTasks += addWith.sumNumCopTasks
+	if sumTo.maxCopProcessTime < addWith.maxCopProcessTime {
+		sumTo.maxCopProcessTime = addWith.maxCopProcessTime
+	}
+	if sumTo.maxCopWaitTime < addWith.maxCopWaitTime {
+		sumTo.maxCopWaitTime = addWith.maxCopWaitTime
+	}
+	// TiKV
+	sumTo.sumProcessTime += addWith.sumProcessTime
+	sumTo.sumWaitTime += addWith.sumWaitTime
+	sumTo.sumBackoffTime += addWith.sumBackoffTime
+	sumTo.sumTotalKeys += addWith.sumTotalKeys
+	sumTo.sumProcessedKeys += addWith.sumProcessedKeys
+	sumTo.sumRocksdbDeleteSkippedCount += addWith.sumRocksdbDeleteSkippedCount
+	sumTo.sumRocksdbKeySkippedCount += addWith.sumRocksdbKeySkippedCount
+	sumTo.sumRocksdbBlockCacheHitCount += addWith.sumRocksdbBlockCacheHitCount
+	sumTo.sumRocksdbBlockReadCount += addWith.sumRocksdbBlockReadCount
+	sumTo.sumRocksdbBlockReadByte += addWith.sumRocksdbBlockReadByte
+	if sumTo.maxProcessTime < addWith.maxProcessTime {
+		sumTo.maxProcessTime = addWith.maxProcessTime
+	}
+	if sumTo.maxWaitTime < addWith.maxWaitTime {
+		sumTo.maxWaitTime = addWith.maxWaitTime
+	}
+	if sumTo.maxBackoffTime < addWith.maxBackoffTime {
+		sumTo.maxBackoffTime = addWith.maxBackoffTime
+	}
+	if sumTo.maxTotalKeys < addWith.maxTotalKeys {
+		sumTo.maxTotalKeys = addWith.maxTotalKeys
+	}
+	if sumTo.maxProcessedKeys < addWith.maxProcessedKeys {
+		sumTo.maxProcessedKeys = addWith.maxProcessedKeys
+	}
+	if sumTo.maxRocksdbBlockReadByte < addWith.maxRocksdbBlockReadByte {
+		sumTo.maxRocksdbBlockReadByte = addWith.maxRocksdbBlockReadByte
+	}
+	if sumTo.maxRocksdbBlockCacheHitCount < addWith.maxRocksdbBlockCacheHitCount {
+		sumTo.maxRocksdbBlockCacheHitCount = addWith.maxRocksdbBlockCacheHitCount
+	}
+	if sumTo.maxRocksdbBlockReadCount < addWith.maxRocksdbBlockReadCount {
+		sumTo.maxRocksdbBlockReadCount = addWith.maxRocksdbBlockReadCount
+	}
+	if sumTo.maxRocksdbDeleteSkippedCount < addWith.maxRocksdbDeleteSkippedCount {
+		sumTo.maxRocksdbDeleteSkippedCount = addWith.maxRocksdbDeleteSkippedCount
+	}
+	if sumTo.maxRocksdbKeySkippedCount < addWith.maxRocksdbKeySkippedCount {
+		sumTo.maxRocksdbKeySkippedCount = addWith.maxRocksdbKeySkippedCount
+	}
+	// txn
+	sumTo.commitCount += addWith.commitCount
+	sumTo.sumGetCommitTsTime += addWith.sumGetCommitTsTime
+	sumTo.sumPrewriteTime += addWith.sumPrewriteTime
+	sumTo.sumCommitTime += addWith.sumCommitTime
+	sumTo.sumLocalLatchTime += addWith.sumLocalLatchTime
+	sumTo.sumCommitBackoffTime += addWith.sumCommitBackoffTime
+	sumTo.sumResolveLockTime += addWith.sumResolveLockTime
+	sumTo.sumWriteKeys += addWith.sumWriteKeys
+	sumTo.sumWriteSize += addWith.sumWriteSize
+	sumTo.sumPrewriteRegionNum += addWith.sumPrewriteRegionNum
+	sumTo.sumTxnRetry += addWith.sumTxnRetry
+	sumTo.sumBackoffTimes += sumTo.sumBackoffTimes
+	if sumTo.maxGetCommitTsTime < addWith.maxGetCommitTsTime {
+		sumTo.maxGetCommitTsTime = addWith.maxGetCommitTsTime
+	}
+	if sumTo.maxPrewriteTime < addWith.maxPrewriteTime {
+		sumTo.maxPrewriteTime = addWith.maxPrewriteTime
+	}
+	if sumTo.maxCommitTime < addWith.maxCommitTime {
+		sumTo.maxCommitTime = addWith.maxCommitTime
+	}
+	if sumTo.maxLocalLatchTime < addWith.maxLocalLatchTime {
+		sumTo.maxLocalLatchTime = addWith.maxLocalLatchTime
+	}
+	if sumTo.maxCommitBackoffTime < addWith.maxCommitBackoffTime {
+		sumTo.maxCommitBackoffTime = addWith.maxCommitBackoffTime
+	}
+	if sumTo.maxResolveLockTime < addWith.maxResolveLockTime {
+		sumTo.maxResolveLockTime = addWith.maxResolveLockTime
+	}
+	if sumTo.maxWriteKeys < addWith.maxWriteKeys {
+		sumTo.maxWriteKeys = addWith.maxWriteKeys
+	}
+	if sumTo.maxWriteSize < addWith.maxWriteSize {
+		sumTo.maxWriteSize = addWith.maxWriteSize
+	}
+	if sumTo.maxPrewriteRegionNum < sumTo.maxPrewriteRegionNum {
+		sumTo.maxPrewriteRegionNum = addWith.maxPrewriteRegionNum
+	}
+	if sumTo.maxTxnRetry < addWith.maxTxnRetry {
+		sumTo.maxTxnRetry = addWith.maxTxnRetry
+	}
+	// other
+	sumTo.sumMem += addWith.sumMem
+	sumTo.sumDisk += addWith.sumDisk
+	sumTo.sumAffectedRows += addWith.sumAffectedRows
+	sumTo.sumKVTotal += addWith.sumKVTotal
+	sumTo.sumPDTotal += addWith.sumPDTotal
+	sumTo.sumBackoffTotal += addWith.sumBackoffTotal
+	sumTo.sumWriteSQLRespTotal += addWith.sumWriteSQLRespTotal
+	if sumTo.maxMem < addWith.maxMem {
+		sumTo.maxMem = addWith.maxMem
+	}
+	if sumTo.maxDisk < addWith.maxDisk {
+		sumTo.maxDisk = addWith.maxDisk
+	}
+	// plan cache
+	sumTo.planCacheHits += addWith.planCacheHits
+	// pessimistic execution retry information
+	sumTo.execRetryCount += addWith.execRetryCount
+	sumTo.execRetryTime += addWith.execRetryTime
+}