*: decouple the dependency between server and mcs (tikv#5933)

ref tikv#5837

Signed-off-by: lhy1024 <admin@liudos.us>

cmd/pd-server/main.go

@@ -25,7 +25,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/log"
 	"github.com/tikv/pd/pkg/autoscaling"
-	basicsvr "github.com/tikv/pd/pkg/basic_server"
+	bs "github.com/tikv/pd/pkg/basicserver"
 	"github.com/tikv/pd/pkg/dashboard"
 	"github.com/tikv/pd/pkg/errs"
 	"github.com/tikv/pd/pkg/swaggerserver"
@@ -36,13 +36,8 @@ import (
 	"github.com/tikv/pd/server/apiv2"
 	"github.com/tikv/pd/server/config"
 	"github.com/tikv/pd/server/join"
-	"go.uber.org/zap"
-
 	"github.com/tikv/pd/server/schedulers"
-
-	// Register Service
-	_ "github.com/tikv/pd/pkg/mcs/registry"
-	_ "github.com/tikv/pd/pkg/mcs/resource_manager/server/install"
+	"go.uber.org/zap"
 )
 
 func main() {
@@ -77,7 +72,7 @@ func main() {
 	}
 }
 
-func createServerWrapper(args []string) (context.Context, context.CancelFunc, basicsvr.Server) {
+func createServerWrapper(args []string) (context.Context, context.CancelFunc, bs.Server) {
 	schedulers.Register()
 	cfg := config.NewConfig()
 	err := cfg.Parse(args)

pkg/autoscaling/service.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net/http"
 
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/pkg/utils/apiutil/serverapi"
 	"github.com/tikv/pd/server"
 	"github.com/unrolled/render"
@@ -27,7 +28,7 @@ import (
 const autoScalingPrefix = "/autoscaling"
 
 var (
-	autoscalingServiceGroup = server.APIServiceGroup{
+	autoscalingServiceGroup = apiutil.APIServiceGroup{
 		Name:       "autoscaling",
 		Version:    "v1alpha",
 		IsCore:     false,
@@ -36,7 +37,7 @@ var (
 )
 
 // NewHandler creates a HTTP handler for auto scaling.
-func NewHandler(_ context.Context, svr *server.Server) (http.Handler, server.APIServiceGroup, error) {
+func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
 	autoScalingHandler := http.NewServeMux()
 	rd := render.New(render.Options{
 		IndentJSON: true,

pkg/basic_server/basic_server.go → pkg/basicserver/basic_server.go

@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package basicsvr
+package server
 
 import (
 	"context"
 	"net/http"
 
+	"github.com/tikv/pd/pkg/member"
 	"go.etcd.io/etcd/clientv3"
 )
 
@@ -27,14 +28,19 @@ type Server interface {
 	Name() string
 	// Context returns the context of server.
 	Context() context.Context
-
 	// Run runs the server.
 	Run() error
 	// Close closes the server.
 	Close()
-
 	// GetClient returns builtin etcd client.
 	GetClient() *clientv3.Client
 	// GetHTTPClient returns builtin http client.
 	GetHTTPClient() *http.Client
+	// AddStartCallback adds a callback in the startServer phase.
+	AddStartCallback(callbacks ...func())
+	// TODO: replace these two methods with `primary` function without etcd server dependency.
+	// GetMember returns the member information.
+	GetMember() *member.Member
+	// AddLeaderCallback adds a callback in the leader campaign phase.
+	AddLeaderCallback(callbacks ...func(context.Context))
 }

pkg/dashboard/dashboard.go

@@ -30,18 +30,19 @@ import (
 	"github.com/tikv/pd/pkg/dashboard/distroutil"
 	"github.com/tikv/pd/pkg/dashboard/keyvisual"
 	ui "github.com/tikv/pd/pkg/dashboard/uiserver"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/server"
 )
 
 var (
-	apiServiceGroup = server.APIServiceGroup{
+	apiServiceGroup = apiutil.APIServiceGroup{
 		Name:       "dashboard-api",
 		Version:    "v1",
 		IsCore:     false,
 		PathPrefix: config.APIPathPrefix,
 	}
 
-	uiServiceGroup = server.APIServiceGroup{
+	uiServiceGroup = apiutil.APIServiceGroup{
 		Name:       "dashboard-ui",
 		Version:    "v1",
 		IsCore:     false,
@@ -68,7 +69,7 @@ func GetServiceBuilders() []server.HandlerBuilder {
 	// The order of execution must be sequential.
 	return []server.HandlerBuilder{
 		// Dashboard API Service
-		func(ctx context.Context, srv *server.Server) (http.Handler, server.APIServiceGroup, error) {
+		func(ctx context.Context, srv *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
 			distroutil.MustLoadAndReplaceStrings()
 
 			if cfg, err = adapter.GenDashboardConfig(srv); err != nil {
@@ -98,7 +99,7 @@ func GetServiceBuilders() []server.HandlerBuilder {
 			return apiserver.Handler(s), apiServiceGroup, nil
 		},
 		// Dashboard UI
-		func(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) {
+		func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
 			if err != nil {
 				return nil, uiServiceGroup, err
 			}

pkg/dashboard/without_dashboard.go

@@ -25,11 +25,12 @@ import (
 
 	"github.com/pingcap/tidb-dashboard/pkg/config"
 
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/server"
 )
 
 var (
-	serviceGroup = server.APIServiceGroup{
+	serviceGroup = apiutil.APIServiceGroup{
 		Name:       "dashboard",
 		Version:    "v1",
 		IsCore:     false,
@@ -43,7 +44,7 @@ func SetCheckInterval(time.Duration) {}
 // GetServiceBuilders returns a empty Dashboard Builder
 func GetServiceBuilders() []server.HandlerBuilder {
 	return []server.HandlerBuilder{
-		func(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) {
+		func(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
 			handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				_, _ = io.WriteString(w, "Dashboard is not built.\n")
 			})

pkg/mcs/registry/registry.go

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 // Package registry is used to register the services.
-// TODO: Remove the `pd/server` dependencies
 // TODO: Use the `uber/fx` to manage the lifecycle of services.
 package registry
 
@@ -22,18 +21,18 @@ import (
 	"net/http"
 
 	"github.com/pingcap/log"
-	"github.com/tikv/pd/server"
+	bs "github.com/tikv/pd/pkg/basicserver"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 )
 
 var (
 	// ServerServiceRegistry is the global grpc service registry.
-	ServerServiceRegistry = newServiceRegistry()
+	ServerServiceRegistry = NewServerServiceRegistry()
 )
 
 // ServiceBuilder is a function that creates a grpc service.
-type ServiceBuilder func(*server.Server) RegistrableService
+type ServiceBuilder func(bs.Server) RegistrableService
 
 // RegistrableService is the interface that should wraps the RegisterService method.
 type RegistrableService interface {
@@ -48,7 +47,8 @@ type ServiceRegistry struct {
 	services map[string]RegistrableService
 }
 
-func newServiceRegistry() *ServiceRegistry {
+// NewServerServiceRegistry creates a new ServiceRegistry.
+func NewServerServiceRegistry() *ServiceRegistry {
 	return &ServiceRegistry{
 		builders: make(map[string]ServiceBuilder),
 		services: make(map[string]RegistrableService),
@@ -60,7 +60,7 @@ func createServiceName(prefix, name string) string {
 }
 
 // InstallAllGRPCServices installs all registered grpc services.
-func (r *ServiceRegistry) InstallAllGRPCServices(srv *server.Server, g *grpc.Server) {
+func (r *ServiceRegistry) InstallAllGRPCServices(srv bs.Server, g *grpc.Server) {
 	prefix := srv.Name()
 	for name, builder := range r.builders {
 		serviceName := createServiceName(prefix, name)
@@ -77,7 +77,7 @@ func (r *ServiceRegistry) InstallAllGRPCServices(srv *server.Server, g *grpc.Ser
 }
 
 // InstallAllRESTHandler installs all registered REST services.
-func (r *ServiceRegistry) InstallAllRESTHandler(srv *server.Server, h map[string]http.Handler) {
+func (r *ServiceRegistry) InstallAllRESTHandler(srv bs.Server, h map[string]http.Handler) {
 	prefix := srv.Name()
 	for name, builder := range r.builders {
 		serviceName := createServiceName(prefix, name)
@@ -94,12 +94,6 @@ func (r *ServiceRegistry) InstallAllRESTHandler(srv *server.Server, h map[string
 }
 
 // RegisterService registers a grpc service.
-func (r ServiceRegistry) RegisterService(name string, service ServiceBuilder) {
+func (r *ServiceRegistry) RegisterService(name string, service ServiceBuilder) {
 	r.builders[name] = service
 }
-
-func init() {
-	server.NewServiceRegistry = func() server.ServiceRegistry {
-		return ServerServiceRegistry
-	}
-}

pkg/mcs/resource_manager/server/apis/v1/api.go

@@ -23,14 +23,14 @@ import (
 	"github.com/gin-gonic/gin"
 	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
 	rmserver "github.com/tikv/pd/pkg/mcs/resource_manager/server"
-	"github.com/tikv/pd/server"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 )
 
 // APIPathPrefix is the prefix of the API path.
 const APIPathPrefix = "/resource-manager/api/v1/"
 
 var (
-	apiServiceGroup = server.APIServiceGroup{
+	apiServiceGroup = apiutil.APIServiceGroup{
 		Name:       "resource-manager",
 		Version:    "v1",
 		IsCore:     false,
@@ -39,7 +39,7 @@ var (
 )
 
 func init() {
-	rmserver.SetUpRestHandler = func(srv *rmserver.Service) (http.Handler, server.APIServiceGroup) {
+	rmserver.SetUpRestHandler = func(srv *rmserver.Service) (http.Handler, apiutil.APIServiceGroup) {
 		s := NewService(srv)
 		return s.handler(), apiServiceGroup
 	}

pkg/mcs/resource_manager/server/grpc_service.go

@@ -23,8 +23,9 @@ import (
 	"github.com/pingcap/errors"
 	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
 	"github.com/pingcap/log"
+	bs "github.com/tikv/pd/pkg/basicserver"
 	"github.com/tikv/pd/pkg/mcs/registry"
-	"github.com/tikv/pd/server"
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -39,8 +40,8 @@ var (
 var _ rmpb.ResourceManagerServer = (*Service)(nil)
 
 // SetUpRestHandler is a hook to sets up the REST service.
-var SetUpRestHandler = func(srv *Service) (http.Handler, server.APIServiceGroup) {
-	return dummyRestService{}, server.APIServiceGroup{}
+var SetUpRestHandler = func(srv *Service) (http.Handler, apiutil.APIServiceGroup) {
+	return dummyRestService{}, apiutil.APIServiceGroup{}
 }
 
 type dummyRestService struct{}
@@ -58,7 +59,7 @@ type Service struct {
 }
 
 // NewService creates a new resource manager service.
-func NewService(svr *server.Server) registry.RegistrableService {
+func NewService(svr bs.Server) registry.RegistrableService {
 	manager := NewManager(svr)
 
 	return &Service{
@@ -75,7 +76,7 @@ func (s *Service) RegisterGRPCService(g *grpc.Server) {
 // RegisterRESTHandler registers the service to REST server.
 func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler) {
 	handler, group := SetUpRestHandler(s)
-	server.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler)
+	apiutil.RegisterUserDefinedHandlers(userDefineHandlers, &group, handler)
 }
 
 // GetManager returns the resource manager.

pkg/mcs/resource_manager/server/manager.go

@@ -26,10 +26,10 @@ import (
 	"github.com/pingcap/failpoint"
 	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
 	"github.com/pingcap/log"
+	bs "github.com/tikv/pd/pkg/basicserver"
 	"github.com/tikv/pd/pkg/member"
 	"github.com/tikv/pd/pkg/storage/endpoint"
 	"github.com/tikv/pd/pkg/storage/kv"
-	"github.com/tikv/pd/server"
 	"go.uber.org/zap"
 )
 
@@ -50,7 +50,7 @@ type Manager struct {
 }
 
 // NewManager returns a new Manager.
-func NewManager(srv *server.Server) *Manager {
+func NewManager(srv bs.Server) *Manager {
 	m := &Manager{
 		member: &member.Member{},
 		groups: make(map[string]*ResourceGroup),

pkg/mcs/tso/server/server.go

@@ -23,25 +23,21 @@ import (
 	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/log"
-	basicsvr "github.com/tikv/pd/pkg/basic_server"
+	bs "github.com/tikv/pd/pkg/basicserver"
 	"github.com/tikv/pd/pkg/errs"
 	"github.com/tikv/pd/pkg/tso"
 	"github.com/tikv/pd/pkg/utils/logutil"
 	"github.com/tikv/pd/pkg/utils/metricutil"
 	"go.etcd.io/etcd/clientv3"
 )
 
-// If server doesn't implement all methods of basicsvr.Server, this line will result in a clear
-// error message like "*Server does not implement basicsvr.Server (missing Method method)"
-var _ basicsvr.Server = (*Server)(nil)
-
-// Server is the TSO server, and it implements basicsvr.Server.
+// Server is the TSO server, and it implements bs.Server.
 // nolint
 type Server struct {
 	ctx context.Context
 }
 
-// TODO: Implement the following methods defined in basicsvr.Server
+// TODO: Implement the following methods defined in bs.Server
 
 // Name returns the unique etcd Name for this server in etcd cluster.
 func (s *Server) Name() string {
@@ -73,7 +69,7 @@ func (s *Server) GetHTTPClient() *http.Client {
 }
 
 // CreateServerWrapper encapsulates the configuration/log/metrics initialization and create the server
-func CreateServerWrapper(args []string) (context.Context, context.CancelFunc, basicsvr.Server) {
+func CreateServerWrapper(args []string) (context.Context, context.CancelFunc, bs.Server) {
 	cfg := tso.NewConfig()
 	err := cfg.Parse(os.Args[1:])
 

pkg/swaggerserver/swaggerserver.go

@@ -18,13 +18,14 @@ import (
 	"context"
 	"net/http"
 
+	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/server"
 )
 
 const swaggerPrefix = "/swagger/"
 
 var (
-	swaggerServiceGroup = server.APIServiceGroup{
+	swaggerServiceGroup = apiutil.APIServiceGroup{
 		Name:       "swagger",
 		Version:    "v1",
 		IsCore:     false,
@@ -33,7 +34,7 @@ var (
 )
 
 // NewHandler creates a HTTP handler for Swagger.
-func NewHandler(context.Context, *server.Server) (http.Handler, server.APIServiceGroup, error) {
+func NewHandler(context.Context, *server.Server) (http.Handler, apiutil.APIServiceGroup, error) {
 	swaggerHandler := http.NewServeMux()
 	swaggerHandler.Handle(swaggerPrefix, handler())
 	return swaggerHandler, swaggerServiceGroup, nil