Merge remote-tracking branch 'origin/master' into feat/tiproxy-conpro…

…f-debug

pkg/apiserver/apiserver.go

@@ -32,17 +32,21 @@ import (
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/user/sso"
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/user/sso/ssoauth"
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/visualplan"
+	"github.com/pingcap/tidb-dashboard/pkg/scheduling"
 	"github.com/pingcap/tidb-dashboard/pkg/ticdc"
 	"github.com/pingcap/tidb-dashboard/pkg/tiflash"
 	"github.com/pingcap/tidb-dashboard/pkg/tiproxy"
+	"github.com/pingcap/tidb-dashboard/pkg/tso"
 	"github.com/pingcap/tidb-dashboard/pkg/utils/version"
 	"github.com/pingcap/tidb-dashboard/util/client/httpclient"
 	"github.com/pingcap/tidb-dashboard/util/client/pdclient"
+	"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
 	"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
+	"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
 	"github.com/pingcap/tidb-dashboard/util/featureflag"
 	"github.com/pingcap/tidb-dashboard/util/rest"
 
@@ -113,6 +117,8 @@ var Modules = fx.Options(
 		httpc.NewHTTPClient,
 		pd.NewEtcdClient,
 		pd.NewPDClient,
+		tso.NewTSOClient,
+		scheduling.NewSchedulingClient,
 		config.NewDynamicConfigManager,
 		tidb.NewTiDBClient,
 		tikv.NewTiKVClient,
@@ -199,6 +205,8 @@ func newClients(lc fx.Lifecycle, config *config.Config) (
 	pdClient *pdclient.APIClient,
 	ticdcClient *ticdcclient.StatusClient,
 	tiproxyClient *tiproxyclient.StatusClient,
+	tsoClient *tsoclient.StatusClient,
+	schedulingClient *schedulingclient.StatusClient,
 ) {
 	httpConfig := httpclient.Config{
 		TLSConfig: config.ClusterTLSConfig,
@@ -209,6 +217,8 @@ func newClients(lc fx.Lifecycle, config *config.Config) (
 	pdClient = pdclient.NewAPIClient(httpConfig)
 	ticdcClient = ticdcclient.NewStatusClient(httpConfig)
 	tiproxyClient = tiproxyclient.NewStatusClient(httpConfig)
+	tsoClient = tsoclient.NewStatusClient(httpConfig)
+	schedulingClient = schedulingclient.NewStatusClient(httpConfig)
 	lc.Append(fx.Hook{
 		OnStart: func(ctx context.Context) error {
 			dbClient.SetDefaultCtx(ctx)

pkg/apiserver/clusterinfo/host.go

@@ -60,6 +60,22 @@ func (s *Service) fetchAllInstanceHosts() ([]string, error) {
 		allHostsMap[i.IP] = struct{}{}
 	}
 
+	tsoInfo, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		return nil, err
+	}
+	for _, i := range tsoInfo {
+		allHostsMap[i.IP] = struct{}{}
+	}
+
+	schedulingInfo, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		return nil, err
+	}
+	for _, i := range schedulingInfo {
+		allHostsMap[i.IP] = struct{}{}
+	}
+
 	allHosts := lo.Keys(allHostsMap)
 	sort.Strings(allHosts)
 

pkg/apiserver/clusterinfo/hostinfo/hostinfo.go

@@ -53,7 +53,7 @@ type InfoMap = map[string]*Info
 var clusterTableQueryTemplate = template.Must(template.New("").Parse(`
 SELECT
 	*,
-	FIELD(LOWER(A.TYPE), 'tiflash', 'tikv', 'pd', 'tidb', 'tiproxy') AS _ORDER
+	FIELD(LOWER(A.TYPE), 'tiflash', 'tikv', 'pd', 'tidb', 'tiproxy', 'tso', 'scheduling') AS _ORDER
 FROM (
 	SELECT
 		TYPE, INSTANCE, DEVICE_TYPE, DEVICE_NAME, JSON_OBJECTAGG(NAME, VALUE) AS JSON_VALUE

pkg/apiserver/clusterinfo/service.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -59,6 +60,8 @@ func RegisterRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	endpoint.DELETE("/tidb/:address", s.deleteTiDBTopology)
 	endpoint.GET("/store", s.getStoreTopology)
 	endpoint.GET("/pd", s.getPDTopology)
+	endpoint.GET("/tso", s.getTSOTopology)
+	endpoint.GET("/scheduling", s.getSchedulingTopology)
 	endpoint.GET("/alertmanager", s.getAlertManagerTopology)
 	endpoint.GET("/alertmanager/:address/count", s.getAlertManagerCounts)
 	endpoint.GET("/grafana", s.getGrafanaTopology)
@@ -156,6 +159,46 @@ func (s *Service) getTiProxyTopology(c *gin.Context) {
 	c.JSON(http.StatusOK, instances)
 }
 
+// @ID getTSOTopology
+// @Summary Get all TSO instances
+// @Success 200 {array} topology.TSOInfo
+// @Router /topology/tso [get]
+// @Security JwtAuth
+// @Failure 401 {object} rest.ErrorResponse
+func (s *Service) getTSOTopology(c *gin.Context) {
+	instances, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		// TODO: refine later
+		if strings.Contains(err.Error(), "status code 404") {
+			rest.Error(c, rest.ErrNotFound.Wrap(err, "api not found"))
+		} else {
+			rest.Error(c, err)
+		}
+		return
+	}
+	c.JSON(http.StatusOK, instances)
+}
+
+// @ID getSchedulingTopology
+// @Summary Get all Scheduling instances
+// @Success 200 {array} topology.SchedulingInfo
+// @Router /topology/scheduling [get]
+// @Security JwtAuth
+// @Failure 401 {object} rest.ErrorResponse
+func (s *Service) getSchedulingTopology(c *gin.Context) {
+	instances, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		// TODO: refine later
+		if strings.Contains(err.Error(), "status code 404") {
+			rest.Error(c, rest.ErrNotFound.Wrap(err, "api not found"))
+		} else {
+			rest.Error(c, err)
+		}
+		return
+	}
+	c.JSON(http.StatusOK, instances)
+}
+
 type StoreTopologyResponse struct {
 	TiKV    []topology.StoreInfo `json:"tikv"`
 	TiFlash []topology.StoreInfo `json:"tiflash"`

pkg/apiserver/clusterinfo/statistics.go

@@ -77,6 +77,8 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
 	infoByIk["tiflash"] = newInstanceKindImmediateInfo()
 	infoByIk["ticdc"] = newInstanceKindImmediateInfo()
 	infoByIk["tiproxy"] = newInstanceKindImmediateInfo()
+	infoByIk["tso"] = newInstanceKindImmediateInfo()
+	infoByIk["scheduling"] = newInstanceKindImmediateInfo()
 
 	// Fill from topology info
 	pdInfo, err := topology.FetchPDTopology(s.params.PDClient)
@@ -135,6 +137,26 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
 		globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
 		infoByIk["tiproxy"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
 	}
+	tsoInfo, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		return nil, err
+	}
+	for _, i := range tsoInfo {
+		globalHostsSet[i.IP] = struct{}{}
+		globalVersionsSet[i.Version] = struct{}{}
+		globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
+		infoByIk["tso"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
+	}
+	schedulingInfo, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
+	if err != nil {
+		return nil, err
+	}
+	for _, i := range schedulingInfo {
+		globalHostsSet[i.IP] = struct{}{}
+		globalVersionsSet[i.Version] = struct{}{}
+		globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
+		infoByIk["scheduling"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
+	}
 
 	// Fill from hardware info
 	allHostsInfoMap := make(map[string]*hostinfo.Info)
@@ -198,6 +220,20 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
 			globalFailureHostsSet[i.IP] = struct{}{}
 		}
 	}
+	for _, i := range tsoInfo {
+		if v, ok := globalInfo.hosts[i.IP]; ok {
+			infoByIk["tso"].hosts[i.IP] = v
+		} else {
+			globalFailureHostsSet[i.IP] = struct{}{}
+		}
+	}
+	for _, i := range schedulingInfo {
+		if v, ok := globalInfo.hosts[i.IP]; ok {
+			infoByIk["scheduling"].hosts[i.IP] = v
+		} else {
+			globalFailureHostsSet[i.IP] = struct{}{}
+		}
+	}
 
 	// Generate result..
 	versions := lo.Keys(globalVersionsSet)

pkg/apiserver/debugapi/endpoint/payload.go

@@ -18,11 +18,13 @@ import (
 	"github.com/pingcap/tidb-dashboard/pkg/utils/topology"
 	"github.com/pingcap/tidb-dashboard/util/client/httpclient"
 	"github.com/pingcap/tidb-dashboard/util/client/pdclient"
+	"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
 	"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
+	"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
 	"github.com/pingcap/tidb-dashboard/util/rest"
 	"github.com/pingcap/tidb-dashboard/util/topo"
 )
@@ -38,12 +40,14 @@ type RequestPayload struct {
 }
 
 type HTTPClients struct {
-	PDAPIClient         *pdclient.APIClient
-	TiDBStatusClient    *tidbclient.StatusClient
-	TiKVStatusClient    *tikvclient.StatusClient
-	TiFlashStatusClient *tiflashclient.StatusClient
-	TiCDCStatusClient   *ticdcclient.StatusClient
-	TiProxyStatusClient *tiproxyclient.StatusClient
+	PDAPIClient            *pdclient.APIClient
+	TiDBStatusClient       *tidbclient.StatusClient
+	TiKVStatusClient       *tikvclient.StatusClient
+	TiFlashStatusClient    *tiflashclient.StatusClient
+	TiCDCStatusClient      *ticdcclient.StatusClient
+	TiProxyStatusClient    *tiproxyclient.StatusClient
+	TSOStatusClient        *tsoclient.StatusClient
+	SchedulingStatusClient *schedulingclient.StatusClient
 }
 
 func (c HTTPClients) GetHTTPClientByNodeKind(kind topo.Kind) *httpclient.Client {
@@ -306,6 +310,36 @@ func (p *ResolvedRequestPayload) verifyEndpoint(ctx context.Context, etcdClient
 		if !matched {
 			return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
 		}
+	case topo.KindTSO:
+		infos, err := topology.FetchTSOTopology(ctx, pdClient)
+		if err != nil {
+			return ErrInvalidEndpoint.Wrap(err, "failed to fetch tso topology")
+		}
+		matched := false
+		for _, info := range infos {
+			if info.IP == p.host && info.Port == uint(p.port) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
+		}
+	case topo.KindScheduling:
+		infos, err := topology.FetchSchedulingTopology(ctx, pdClient)
+		if err != nil {
+			return ErrInvalidEndpoint.Wrap(err, "failed to fetch scheduling topology")
+		}
+		matched := false
+		for _, info := range infos {
+			if info.IP == p.host && info.Port == uint(p.port) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
+		}
 	default:
 		return ErrUnknownComponent.New("Unknown component '%s'", p.api.Component)
 	}

pkg/apiserver/debugapi/service.go

@@ -16,11 +16,13 @@ import (
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/user"
 	"github.com/pingcap/tidb-dashboard/pkg/pd"
 	"github.com/pingcap/tidb-dashboard/util/client/pdclient"
+	"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
 	"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
 	"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
+	"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
 	"github.com/pingcap/tidb-dashboard/util/rest"
 	"github.com/pingcap/tidb-dashboard/util/rest/fileswap"
 )
@@ -37,14 +39,16 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 
 type ServiceParams struct {
 	fx.In
-	PDAPIClient         *pdclient.APIClient
-	TiDBStatusClient    *tidbclient.StatusClient
-	TiKVStatusClient    *tikvclient.StatusClient
-	TiFlashStatusClient *tiflashclient.StatusClient
-	TiCDCStatusClient   *ticdcclient.StatusClient
-	TiProxyStatusClient *tiproxyclient.StatusClient
-	EtcdClient          *clientv3.Client
-	PDClient            *pd.Client
+	PDAPIClient            *pdclient.APIClient
+	TiDBStatusClient       *tidbclient.StatusClient
+	TiKVStatusClient       *tikvclient.StatusClient
+	TiFlashStatusClient    *tiflashclient.StatusClient
+	TiCDCStatusClient      *ticdcclient.StatusClient
+	TiProxyStatusClient    *tiproxyclient.StatusClient
+	EtcdClient             *clientv3.Client
+	PDClient               *pd.Client
+	TSOStatusClient        *tsoclient.StatusClient
+	SchedulingStatusClient *schedulingclient.StatusClient
 }
 
 type Service struct {
@@ -57,12 +61,14 @@ type Service struct {
 
 func newService(p ServiceParams) *Service {
 	httpClients := endpoint.HTTPClients{
-		PDAPIClient:         p.PDAPIClient,
-		TiDBStatusClient:    p.TiDBStatusClient,
-		TiKVStatusClient:    p.TiKVStatusClient,
-		TiFlashStatusClient: p.TiFlashStatusClient,
-		TiCDCStatusClient:   p.TiCDCStatusClient,
-		TiProxyStatusClient: p.TiProxyStatusClient,
+		PDAPIClient:            p.PDAPIClient,
+		TiDBStatusClient:       p.TiDBStatusClient,
+		TiKVStatusClient:       p.TiKVStatusClient,
+		TiFlashStatusClient:    p.TiFlashStatusClient,
+		TiCDCStatusClient:      p.TiCDCStatusClient,
+		TiProxyStatusClient:    p.TiProxyStatusClient,
+		TSOStatusClient:        p.TSOStatusClient,
+		SchedulingStatusClient: p.SchedulingStatusClient,
 	}
 	return &Service{
 		httpClients: httpClients,

pkg/apiserver/model/common_models.go

@@ -10,12 +10,14 @@ import (
 type NodeKind string
 
 const (
-	NodeKindTiDB    NodeKind = "tidb"
-	NodeKindTiKV    NodeKind = "tikv"
-	NodeKindPD      NodeKind = "pd"
-	NodeKindTiFlash NodeKind = "tiflash"
-	NodeKindTiCDC   NodeKind = "ticdc"
-	NodeKindTiProxy NodeKind = "tiproxy"
+	NodeKindTiDB       NodeKind = "tidb"
+	NodeKindTiKV       NodeKind = "tikv"
+	NodeKindPD         NodeKind = "pd"
+	NodeKindTiFlash    NodeKind = "tiflash"
+	NodeKindTiCDC      NodeKind = "ticdc"
+	NodeKindTiProxy    NodeKind = "tiproxy"
+	NodeKindTSO        NodeKind = "tso"
+	NodeKindScheduling NodeKind = "scheduling"
 )
 
 type RequestTargetNode struct {
@@ -35,12 +37,14 @@ func (n *RequestTargetNode) FileName() string {
 }
 
 type RequestTargetStatistics struct {
-	NumTiKVNodes    int `json:"num_tikv_nodes"`
-	NumTiDBNodes    int `json:"num_tidb_nodes"`
-	NumPDNodes      int `json:"num_pd_nodes"`
-	NumTiFlashNodes int `json:"num_tiflash_nodes"`
-	NumTiCDCNodes   int `json:"num_ticdc_nodes"`
-	NumTiProxyNodes int `json:"num_tiproxy_nodes"`
+	NumTiKVNodes       int `json:"num_tikv_nodes"`
+	NumTiDBNodes       int `json:"num_tidb_nodes"`
+	NumPDNodes         int `json:"num_pd_nodes"`
+	NumTiFlashNodes    int `json:"num_tiflash_nodes"`
+	NumTiCDCNodes      int `json:"num_ticdc_nodes"`
+	NumTiProxyNodes    int `json:"num_tiproxy_nodes"`
+	NumTSONodes        int `json:"num_tso_nodes"`
+	NumSchedulingNodes int `json:"num_scheduling_nodes"`
 }
 
 func NewRequestTargetStatisticsFromArray(arr *[]RequestTargetNode) RequestTargetStatistics {
@@ -59,6 +63,10 @@ func NewRequestTargetStatisticsFromArray(arr *[]RequestTargetNode) RequestTarget
 			stats.NumTiCDCNodes++
 		case NodeKindTiProxy:
 			stats.NumTiProxyNodes++
+		case NodeKindTSO:
+			stats.NumTSONodes++
+		case NodeKindScheduling:
+			stats.NumSchedulingNodes++
 		}
 	}
 	return stats