cherry pick master 90ec2d0 (tikv#106)

Signed-off-by: Cabinfever_B <cabinfeveroier@gmail.com>

pkg/mcs/resource_manager/server/manager.go

@@ -37,6 +37,7 @@ const (
 	defaultConsumptionChanSize = 1024
 	metricsCleanupInterval     = time.Minute
 	metricsCleanupTimeout      = 20 * time.Minute
+	metricsAvailableRUInterval = 30 * time.Second
 )
 
 // Manager is the manager of resource group.
@@ -254,8 +255,10 @@ func (m *Manager) persistResourceGroupRunningState() {
 // Receive the consumption and flush it to the metrics.
 func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 	defer logutil.LogPanic()
-	ticker := time.NewTicker(metricsCleanupInterval)
-	defer ticker.Stop()
+	cleanUpTicker := time.NewTicker(metricsCleanupInterval)
+	defer cleanUpTicker.Stop()
+	availableRUTicker := time.NewTicker(metricsAvailableRUInterval)
+	defer availableRUTicker.Stop()
 	for {
 		select {
 		case <-ctx.Done():
@@ -309,7 +312,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 
 			m.consumptionRecord[name] = time.Now()
 
-		case <-ticker.C:
+		case <-cleanUpTicker.C:
 			// Clean up the metrics that have not been updated for a long time.
 			for name, lastTime := range m.consumptionRecord {
 				if time.Since(lastTime) > metricsCleanupTimeout {
@@ -322,9 +325,20 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 					sqlCPUCost.DeleteLabelValues(name)
 					requestCount.DeleteLabelValues(name, readTypeLabel)
 					requestCount.DeleteLabelValues(name, writeTypeLabel)
+					availableRUCounter.DeleteLabelValues(name)
 					delete(m.consumptionRecord, name)
 				}
 			}
+		case <-availableRUTicker.C:
+			m.RLock()
+			for name, group := range m.groups {
+				ru := group.getRUToken()
+				if ru < 0 {
+					ru = 0
+				}
+				availableRUCounter.WithLabelValues(name).Set(ru)
+			}
+			m.RUnlock()
 		}
 	}
 }

pkg/mcs/resource_manager/server/metrics.go

@@ -94,6 +94,14 @@ var (
 			Name:      "request_count",
 			Help:      "The number of read/write requests for all resource groups.",
 		}, []string{resourceGroupNameLabel, typeLabel})
+
+	availableRUCounter = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: ruSubsystem,
+			Name:      "available_ru",
+			Help:      "Counter of the available RU for all resource groups.",
+		}, []string{resourceGroupNameLabel})
 )
 
 func init() {
@@ -105,4 +113,5 @@ func init() {
 	prometheus.MustRegister(kvCPUCost)
 	prometheus.MustRegister(sqlCPUCost)
 	prometheus.MustRegister(requestCount)
+	prometheus.MustRegister(availableRUCounter)
 }

pkg/mcs/resource_manager/server/resource_group.go

@@ -104,6 +104,12 @@ func (rg *ResourceGroup) CheckAndInit() error {
 	return nil
 }
 
+func (rg *ResourceGroup) getRUToken() float64 {
+	rg.Lock()
+	defer rg.Unlock()
+	return rg.RUSettings.RU.Tokens
+}
+
 // PatchSettings patches the resource group settings.
 // Only used to patch the resource group when updating.
 // Note: the tokens is the delta value to patch.