add cluster name to metrics (#64)

pkg/config/proxy.go

@@ -4,8 +4,13 @@ const (
 	MIN_SESSION_TIMEOUT = 600
 )
 
+const (
+	DefaultClusterName = "default"
+)
+
 type Proxy struct {
 	Version      string       `yaml:"version"`
+	Cluster      string       `yaml:"cluster"`
 	ProxyServer  ProxyServer  `yaml:"proxy_server"`
 	AdminServer  AdminServer  `yaml:"admin_server"`
 	Log          Log          `yaml:"log"`

pkg/proxy/metrics/backend.go

@@ -4,9 +4,9 @@ import "github.com/prometheus/client_golang/prometheus"
 
 const (
 	BackendEventIniting = "initing"
-	BackendEventInited = "inited"
+	BackendEventInited  = "inited"
 	BackendEventClosing = "closing"
-	BackendEventClosed = "closed"
+	BackendEventClosed  = "closed"
 )
 
 var (
@@ -16,21 +16,21 @@ var (
 			Subsystem: LabelBackend,
 			Name:      "backend_event_total",
 			Help:      "Counter of backend event.",
-		}, []string{LblNamespace, LblType})
+		}, []string{LblCluster, LblNamespace, LblType})
 
 	BackendQueryCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelBackend,
 			Name:      "b_conn_cnt",
 			Help:      "Counter of backend query count.",
-		}, []string{LblNamespace, LblBackendAddr})
+		}, []string{LblCluster, LblNamespace, LblBackendAddr})
 
 	BackendConnInUseGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelBackend,
 			Name:      "b_conn_in_use",
 			Help:      "Number of backend conn in use.",
-		}, []string{LblNamespace, LblBackendAddr})
+		}, []string{LblCluster, LblNamespace, LblBackendAddr})
 )

pkg/proxy/metrics/metrics.go

@@ -1,6 +1,8 @@
 package metrics
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
 
 const (
 	ModuleWeirProxy = "weirproxy"
@@ -40,26 +42,37 @@ func RetLabel(err error) string {
 	return opFailed
 }
 
-func RegisterProxyMetrics() {
+func RegisterProxyMetrics(cluster string) {
+	curryingLabelsWithLblCluster := map[string]string{LblCluster: cluster}
+
+	PanicCounter = PanicCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(PanicCounter)
+	QueryTotalCounter = QueryTotalCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(QueryTotalCounter)
+	ExecuteErrorCounter = ExecuteErrorCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(ExecuteErrorCounter)
-	prometheus.MustRegister(CriticalErrorCounter)
+	ConnGauge = ConnGauge.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(ConnGauge)
-	prometheus.MustRegister(ServerEventCounter)
-	prometheus.MustRegister(GetTokenDurationHistogram)
-	prometheus.MustRegister(HandShakeErrorCounter)
 
 	// query ctx metrics
+	QueryCtxQueryCounter = QueryCtxQueryCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(QueryCtxQueryCounter)
+	QueryCtxQueryDeniedCounter = QueryCtxQueryDeniedCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(QueryCtxQueryDeniedCounter)
+	QueryCtxQueryDurationHistogram = QueryCtxQueryDurationHistogram.MustCurryWith(curryingLabelsWithLblCluster).(*prometheus.HistogramVec)
 	prometheus.MustRegister(QueryCtxQueryDurationHistogram)
+	QueryCtxGauge = QueryCtxGauge.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(QueryCtxGauge)
+	QueryCtxAttachedConnGauge = QueryCtxAttachedConnGauge.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(QueryCtxAttachedConnGauge)
+	QueryCtxTransactionDuration = QueryCtxTransactionDuration.MustCurryWith(curryingLabelsWithLblCluster).(*prometheus.HistogramVec)
 	prometheus.MustRegister(QueryCtxTransactionDuration)
 
 	// backend metrics
+	BackendEventCounter = BackendEventCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(BackendEventCounter)
+	BackendQueryCounter = BackendQueryCounter.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(BackendQueryCounter)
+	BackendConnInUseGauge = BackendConnInUseGauge.MustCurryWith(curryingLabelsWithLblCluster)
 	prometheus.MustRegister(BackendConnInUseGauge)
 }

pkg/proxy/metrics/queryctx.go

@@ -46,15 +46,15 @@ var (
 			Subsystem: LabelQueryCtx,
 			Name:      "query_total",
 			Help:      "Counter of queries.",
-		}, []string{LblNamespace, LblDb, LblTable, LblSQLType, LblResult})
+		}, []string{LblCluster, LblNamespace, LblDb, LblTable, LblSQLType, LblResult})
 
 	QueryCtxQueryDeniedCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelQueryCtx,
 			Name:      "query_denied",
 			Help:      "Counter of denied queries.",
-		}, []string{LblNamespace, LblDb, LblTable, LblSQLType})
+		}, []string{LblCluster, LblNamespace, LblDb, LblTable, LblSQLType})
 
 	QueryCtxQueryDurationHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
@@ -63,23 +63,23 @@ var (
 			Name:      "handle_query_duration_seconds",
 			Help:      "Bucketed histogram of processing time (s) of handled queries.",
 			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 29), // 0.5ms ~ 1.5days
-		}, []string{LblNamespace, LblDb, LblTable, LblSQLType})
+		}, []string{LblCluster, LblNamespace, LblDb, LblTable, LblSQLType})
 
 	QueryCtxGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelQueryCtx,
 			Name:      "queryctx",
 			Help:      "Number of queryctx (equals to client connection).",
-		}, []string{LblNamespace})
+		}, []string{LblCluster, LblNamespace})
 
 	QueryCtxAttachedConnGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelQueryCtx,
 			Name:      "attached_connections",
 			Help:      "Number of attached backend connections.",
-		}, []string{LblNamespace})
+		}, []string{LblCluster, LblNamespace})
 
 	QueryCtxTransactionDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
@@ -88,7 +88,7 @@ var (
 			Name:      "transaction_duration_seconds",
 			Help:      "Bucketed histogram of a transaction execution duration, including retry.",
 			Buckets:   prometheus.ExponentialBuckets(0.001, 2, 28), // 1ms ~ 1.5days
-		}, []string{LblNamespace, LblDb, LblSQLType})
+		}, []string{LblCluster, LblNamespace, LblDb, LblSQLType})
 )
 
 func GetStmtType(stmt ast.StmtNode) AstStmtType {

pkg/proxy/metrics/server.go

@@ -16,70 +16,37 @@ var (
 			Subsystem: LabelServer,
 			Name:      "panic_total",
 			Help:      "Counter of panic.",
-		}, []string{LblType})
+		}, []string{LblCluster, LblType})
 
 	QueryTotalCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelServer,
 			Name:      "query_total",
 			Help:      "Counter of queries.",
-		}, []string{LblType, LblResult})
+		}, []string{LblCluster, LblType, LblResult})
 
 	ExecuteErrorCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelServer,
 			Name:      "execute_error_total",
 			Help:      "Counter of execute errors.",
-		}, []string{LblType})
+		}, []string{LblCluster, LblType})
 
-	CriticalErrorCounter = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: ModuleWeirProxy,
-			Subsystem: LabelServer,
-			Name:      "critical_error_total",
-			Help:      "Counter of critical errors.",
-		})
-
-	ConnGauge = prometheus.NewGauge(
+	ConnGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: ModuleWeirProxy,
 			Subsystem: LabelServer,
 			Name:      "connections",
 			Help:      "Number of connections.",
-		})
+		}, []string{LblCluster})
 
 	EventStart        = "start"
 	EventGracefulDown = "graceful_shutdown"
 	// Eventkill occurs when the server.Kill() function is called.
-	EventKill          = "kill"
-	EventClose         = "close"
-	ServerEventCounter = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: ModuleWeirProxy,
-			Subsystem: LabelServer,
-			Name:      "event_total",
-			Help:      "Counter of weirproxy-server event.",
-		}, []string{LblType})
-
-	GetTokenDurationHistogram = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: ModuleWeirProxy,
-			Subsystem: LabelServer,
-			Name:      "get_token_duration_seconds",
-			Help:      "Duration (us) for getting token, it should be small until concurrency limit is reached.",
-			Buckets:   prometheus.ExponentialBuckets(1, 2, 30), // 1us ~ 528s
-		})
-
-	HandShakeErrorCounter = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: ModuleWeirProxy,
-			Subsystem: LabelServer,
-			Name:      "handshake_error_total",
-			Help:      "Counter of hand shake error.",
-		},
-	)
+	EventKill  = "kill"
+	EventClose = "close"
 )
 
 // ExecuteErrorToLabel converts an execute error to label.

pkg/proxy/metrics/session.go

@@ -24,6 +24,7 @@ const (
 	LblBatchGet    = "batch_get"
 	LblGet         = "get"
 	LblNamespace   = "namespace"
+	LblCluster     = "cluster"
 
 	LblBackendAddr = "backend_addr"
 )

pkg/proxy/proxy.go

@@ -6,6 +6,7 @@ import (
 	"github.com/pingcap-incubator/weir/pkg/config"
 	"github.com/pingcap-incubator/weir/pkg/configcenter"
 	"github.com/pingcap-incubator/weir/pkg/proxy/driver"
+	"github.com/pingcap-incubator/weir/pkg/proxy/metrics"
 	"github.com/pingcap-incubator/weir/pkg/proxy/namespace"
 	"github.com/pingcap-incubator/weir/pkg/proxy/server"
 )
@@ -22,6 +23,9 @@ func supplementProxyConfig(cfg *config.Proxy) *config.Proxy {
 	if cfg.ProxyServer.SessionTimeout <= config.MIN_SESSION_TIMEOUT {
 		cfg.ProxyServer.SessionTimeout = config.MIN_SESSION_TIMEOUT
 	}
+	if cfg.Cluster == "" {
+		cfg.Cluster = config.DefaultClusterName
+	}
 	return cfg
 }
 
@@ -32,6 +36,7 @@ func NewProxy(cfg *config.Proxy) *Proxy {
 }
 
 func (p *Proxy) Init() error {
+	metrics.RegisterProxyMetrics(p.cfg.Cluster)
 	cc, err := configcenter.CreateConfigCenter(p.cfg.ConfigCenter)
 	if err != nil {
 		return err

pkg/proxy/server/conn.go

@@ -51,38 +51,28 @@ const (
 )
 
 var (
-	queryTotalCountOk = [...]prometheus.Counter{
-		mysql.ComSleep:            metrics.QueryTotalCounter.WithLabelValues("Sleep", "OK"),
-		mysql.ComQuit:             metrics.QueryTotalCounter.WithLabelValues("Quit", "OK"),
-		mysql.ComInitDB:           metrics.QueryTotalCounter.WithLabelValues("InitDB", "OK"),
-		mysql.ComQuery:            metrics.QueryTotalCounter.WithLabelValues("Query", "OK"),
-		mysql.ComPing:             metrics.QueryTotalCounter.WithLabelValues("Ping", "OK"),
-		mysql.ComFieldList:        metrics.QueryTotalCounter.WithLabelValues("FieldList", "OK"),
-		mysql.ComStmtPrepare:      metrics.QueryTotalCounter.WithLabelValues("StmtPrepare", "OK"),
-		mysql.ComStmtExecute:      metrics.QueryTotalCounter.WithLabelValues("StmtExecute", "OK"),
-		mysql.ComStmtFetch:        metrics.QueryTotalCounter.WithLabelValues("StmtFetch", "OK"),
-		mysql.ComStmtClose:        metrics.QueryTotalCounter.WithLabelValues("StmtClose", "OK"),
-		mysql.ComStmtSendLongData: metrics.QueryTotalCounter.WithLabelValues("StmtSendLongData", "OK"),
-		mysql.ComStmtReset:        metrics.QueryTotalCounter.WithLabelValues("StmtReset", "OK"),
-		mysql.ComSetOption:        metrics.QueryTotalCounter.WithLabelValues("SetOption", "OK"),
-	}
-	queryTotalCountErr = [...]prometheus.Counter{
-		mysql.ComSleep:            metrics.QueryTotalCounter.WithLabelValues("Sleep", "Error"),
-		mysql.ComQuit:             metrics.QueryTotalCounter.WithLabelValues("Quit", "Error"),
-		mysql.ComInitDB:           metrics.QueryTotalCounter.WithLabelValues("InitDB", "Error"),
-		mysql.ComQuery:            metrics.QueryTotalCounter.WithLabelValues("Query", "Error"),
-		mysql.ComPing:             metrics.QueryTotalCounter.WithLabelValues("Ping", "Error"),
-		mysql.ComFieldList:        metrics.QueryTotalCounter.WithLabelValues("FieldList", "Error"),
-		mysql.ComStmtPrepare:      metrics.QueryTotalCounter.WithLabelValues("StmtPrepare", "Error"),
-		mysql.ComStmtExecute:      metrics.QueryTotalCounter.WithLabelValues("StmtExecute", "Error"),
-		mysql.ComStmtFetch:        metrics.QueryTotalCounter.WithLabelValues("StmtFetch", "Error"),
-		mysql.ComStmtClose:        metrics.QueryTotalCounter.WithLabelValues("StmtClose", "Error"),
-		mysql.ComStmtSendLongData: metrics.QueryTotalCounter.WithLabelValues("StmtSendLongData", "Error"),
-		mysql.ComStmtReset:        metrics.QueryTotalCounter.WithLabelValues("StmtReset", "Error"),
-		mysql.ComSetOption:        metrics.QueryTotalCounter.WithLabelValues("SetOption", "Error"),
+	mysqlComMapping = [...]string{
+		mysql.ComSleep:            "Sleep",
+		mysql.ComQuit:             "Quit",
+		mysql.ComInitDB:           "InitDB",
+		mysql.ComQuery:            "Query",
+		mysql.ComPing:             "Ping",
+		mysql.ComFieldList:        "FieldList",
+		mysql.ComStmtPrepare:      "StmtPrepare",
+		mysql.ComStmtExecute:      "StmtExecute",
+		mysql.ComStmtFetch:        "StmtFetch",
+		mysql.ComStmtClose:        "StmtClose",
+		mysql.ComStmtSendLongData: "StmtSendLongData",
+		mysql.ComStmtReset:        "StmtReset",
+		mysql.ComSetOption:        "SetOption",
 	}
 )
 
+const (
+	mysqlComOK    = "OK"
+	mysqlComError = "Error"
+)
+
 // clientConn represents a connection between server and client, it maintains connection specific state,
 // handles client query.
 type clientConn struct {
@@ -183,7 +173,6 @@ func (cc *clientConn) Run(ctx context.Context) {
 				logutil.Logger(ctx).Error("result undetermined, close this connection", zap.Error(err))
 				return
 			} else if terror.ErrCritical.Equal(err) {
-				metrics.CriticalErrorCounter.Add(1)
 				logutil.Logger(ctx).Fatal("critical error, stop the server", zap.Error(err))
 			}
 			var txnMode string
@@ -287,7 +276,7 @@ func (cc *clientConn) Close() error {
 }
 
 func closeConn(cc *clientConn, connections int) error {
-	metrics.ConnGauge.Set(float64(connections))
+	metrics.ConnGauge.WithLabelValues().Set(float64(connections))
 	err := cc.bufReadConn.Close()
 	terror.Log(err)
 	if cc.ctx != nil {
@@ -326,20 +315,20 @@ func (cc *clientConn) String() string {
 
 // TODO(eastfisher): implement this function
 func (cc *clientConn) addMetrics(cmd byte, startTime time.Time, err error) {
-	var counter prometheus.Counter
-	if err != nil && int(cmd) < len(queryTotalCountErr) {
-		counter = queryTotalCountErr[cmd]
-	} else if err == nil && int(cmd) < len(queryTotalCountOk) {
-		counter = queryTotalCountOk[cmd]
+	getQueryTotalCounter(cmd, err).Inc()
+}
+
+func getQueryTotalCounter(cmd byte, err error) prometheus.Counter {
+	var comTypeStr string
+	if int(cmd) < len(mysqlComMapping) {
+		comTypeStr = mysqlComMapping[cmd]
 	}
-	if counter != nil {
-		counter.Inc()
-	} else {
-		label := strconv.Itoa(int(cmd))
-		if err != nil {
-			metrics.QueryTotalCounter.WithLabelValues(label, "ERROR").Inc()
-		} else {
-			metrics.QueryTotalCounter.WithLabelValues(label, "OK").Inc()
-		}
+	if comTypeStr == "" {
+		comTypeStr = strconv.Itoa(int(cmd))
+	}
+	var comRespStr string = mysqlComOK
+	if err != nil {
+		comRespStr = mysqlComError
 	}
+	return metrics.QueryTotalCounter.WithLabelValues(comTypeStr, comRespStr)
 }