diff --git a/cmd/dm-ctl/main.go b/cmd/dm-ctl/main.go index b0603ba3d8..b651d525e9 100644 --- a/cmd/dm-ctl/main.go +++ b/cmd/dm-ctl/main.go @@ -14,6 +14,7 @@ package main import ( + "context" "flag" "fmt" "io" @@ -195,6 +196,10 @@ func interactionMode() { } }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go common.SyncMasterEndpoints(ctx) + loop() fmt.Println("dmctl exit") diff --git a/dm/ctl/common/config.go b/dm/ctl/common/config.go index ac496b72df..7ee635d657 100644 --- a/dm/ctl/common/config.go +++ b/dm/ctl/common/config.go @@ -18,6 +18,7 @@ import ( "flag" "fmt" "net" + "strings" "time" "github.com/pingcap/dm/dm/config" @@ -39,6 +40,11 @@ const ( Master = "master" // Worker specifies member worker type Worker = "worker" + + dialTimeout = 3 * time.Second + keepaliveTimeout = 3 * time.Second + keepaliveTime = 3 * time.Second + syncMasterEndpointsTime = 3 * time.Second ) // NewConfig creates a new base config for dmctl. @@ -139,7 +145,7 @@ func (c *Config) Parse(arguments []string) (finish bool, err error) { } if c.MasterAddr == "" { - return false, flag.ErrHelp + return false, errors.Errorf("--master-addr not provided, use --help to see help messages") } return false, errors.Trace(c.adjust()) @@ -180,6 +186,11 @@ func (c *Config) adjust() error { // validate host:port format address func validateAddr(addr string) error { - _, _, err := net.SplitHostPort(addr) - return errors.Trace(err) + endpoints := strings.Split(addr, ",") + for _, endpoint := range endpoints { + if _, _, err := net.SplitHostPort(utils.UnwrapScheme(endpoint)); err != nil { + return errors.Trace(err) + } + } + return nil } diff --git a/dm/ctl/common/operate_relay.go b/dm/ctl/common/operate_relay.go index 1a21645ae0..015ad4a78f 100644 --- a/dm/ctl/common/operate_relay.go +++ b/dm/ctl/common/operate_relay.go @@ -23,9 +23,15 @@ import ( func OperateRelay(op pb.RelayOp, workers []string) (*pb.OperateWorkerRelayResponse, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := MasterClient() - return cli.OperateWorkerRelayTask(ctx, &pb.OperateWorkerRelayRequest{ - Op: op, - Sources: workers, - }) + resp := &pb.OperateWorkerRelayResponse{} + err := SendRequest( + ctx, + "OperateWorkerRelayTask", + &pb.OperateWorkerRelayRequest{ + Op: op, + Sources: workers, + }, + &resp, + ) + return resp, err } diff --git a/dm/ctl/common/operate_task.go b/dm/ctl/common/operate_task.go index 49d848b30f..4359591891 100644 --- a/dm/ctl/common/operate_task.go +++ b/dm/ctl/common/operate_task.go @@ -23,10 +23,17 @@ import ( func OperateTask(op pb.TaskOp, name string, sources []string) (*pb.OperateTaskResponse, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := MasterClient() - return cli.OperateTask(ctx, &pb.OperateTaskRequest{ - Op: op, - Name: name, - Sources: sources, - }) + + resp := &pb.OperateTaskResponse{} + err := SendRequest( + ctx, + "OperateTask", + &pb.OperateTaskRequest{ + Op: op, + Name: name, + Sources: sources, + }, + &resp, + ) + return resp, err } diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index 59a042a84e..782784eba4 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -14,10 +14,13 @@ package common import ( + "context" "encoding/json" "fmt" "io/ioutil" + "reflect" "strings" + "sync" "time" "github.com/pingcap/dm/dm/config" @@ -25,6 +28,7 @@ import ( parserpkg "github.com/pingcap/dm/pkg/parser" "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/dm/pkg/utils" + "go.etcd.io/etcd/clientv3" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" @@ -33,13 +37,83 @@ import ( toolutils "github.com/pingcap/tidb-tools/pkg/utils" "github.com/spf13/cobra" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( - masterClient pb.MasterClient globalConfig = &Config{} + ctlClient = &CtlClient{} ) +// CtlClient used to get master client for dmctl +type CtlClient struct { + mu sync.RWMutex + tls *toolutils.TLS + etcdClient *clientv3.Client + conn *grpc.ClientConn + masterClient pb.MasterClient +} + +func (c *CtlClient) updateMasterClient() error { + var ( + err error + conn *grpc.ClientConn + ) + + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil { + c.conn.Close() + } + + endpoints := c.etcdClient.Endpoints() + for _, endpoint := range endpoints { + //nolint:staticcheck + conn, err = grpc.Dial(utils.UnwrapScheme(endpoint), c.tls.ToGRPCDialOption(), grpc.WithBackoffMaxDelay(3*time.Second), grpc.WithBlock(), grpc.WithTimeout(3*time.Second)) + if err == nil { + c.conn = conn + c.masterClient = pb.NewMasterClient(conn) + return nil + } + } + return terror.ErrCtlGRPCCreateConn.AnnotateDelegate(err, "can't connect to %s", strings.Join(endpoints, ",")) +} + +func (c *CtlClient) sendRequest(ctx context.Context, reqName string, req interface{}, respPointer interface{}) error { + c.mu.RLock() + defer c.mu.RUnlock() + + params := []reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(req)} + results := reflect.ValueOf(c.masterClient).MethodByName(reqName).Call(params) + + reflect.ValueOf(respPointer).Elem().Set(results[0]) + errInterface := results[1].Interface() + // nil can't pass type conversion, so we handle it separately + if errInterface == nil { + return nil + } + return errInterface.(error) +} + +// SendRequest send request to master +func SendRequest(ctx context.Context, reqName string, req interface{}, respPointer interface{}) error { + err := ctlClient.sendRequest(ctx, reqName, req, respPointer) + if err == nil || status.Code(err) != codes.Unavailable { + return err + } + + // update master client + err = ctlClient.updateMasterClient() + if err != nil { + return err + } + + // sendRequest again + return ctlClient.sendRequest(ctx, reqName, req, respPointer) +} + // InitUtils inits necessary dmctl utils func InitUtils(cfg *Config) error { globalConfig = cfg @@ -53,13 +127,24 @@ func InitClient(addr string, securityCfg config.Security) error { return terror.ErrCtlInvalidTLSCfg.Delegate(err) } - //nolint:staticcheck - conn, err := grpc.Dial(addr, tls.ToGRPCDialOption(), grpc.WithBackoffMaxDelay(3*time.Second), grpc.WithBlock(), grpc.WithTimeout(3*time.Second)) + endpoints := strings.Split(addr, ",") + etcdClient, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: dialTimeout, + DialKeepAliveTime: keepaliveTime, + DialKeepAliveTimeout: keepaliveTimeout, + TLS: tls.TLSConfig(), + }) if err != nil { - return terror.ErrCtlGRPCCreateConn.AnnotateDelegate(err, "can't connect to %s", addr) + return err } - masterClient = pb.NewMasterClient(conn) - return nil + + ctlClient = &CtlClient{ + tls: tls, + etcdClient: etcdClient, + } + + return ctlClient.updateMasterClient() } // GlobalConfig returns global dmctl config @@ -67,11 +152,6 @@ func GlobalConfig() *Config { return globalConfig } -// MasterClient returns dm-master client -func MasterClient() pb.MasterClient { - return masterClient -} - // PrintLines adds a wrap to support `\n` within `chzyer/readline` func PrintLines(format string, a ...interface{}) { fmt.Println(fmt.Sprintf(format, a...)) @@ -234,3 +314,36 @@ func PrintCmdUsage(cmd *cobra.Command) { fmt.Println("can't output command's usage:", err) } } + +// SyncMasterEndpoints sync masters' endpoints +func SyncMasterEndpoints(ctx context.Context) { + lastClientUrls := []string{} + clientURLs := []string{} + updateF := func() { + clientURLs = clientURLs[:0] + resp, err := ctlClient.etcdClient.MemberList(ctx) + if err != nil { + return + } + + for _, m := range resp.Members { + clientURLs = append(clientURLs, m.GetClientURLs()...) + } + if utils.NonRepeatStringsEqual(clientURLs, lastClientUrls) { + return + } + ctlClient.etcdClient.SetEndpoints(clientURLs...) + lastClientUrls = make([]string, len(clientURLs)) + copy(lastClientUrls, clientURLs) + } + + for { + updateF() + + select { + case <-ctx.Done(): + return + case <-time.After(syncMasterEndpointsTime): + } + } +} diff --git a/dm/ctl/master/check_task.go b/dm/ctl/master/check_task.go index c0f783c305..736569b0d6 100644 --- a/dm/ctl/master/check_task.go +++ b/dm/ctl/master/check_task.go @@ -52,10 +52,16 @@ func checkTaskFunc(cmd *cobra.Command, _ []string) (err error) { defer cancel() // start task - cli := common.MasterClient() - resp, err := cli.CheckTask(ctx, &pb.CheckTaskRequest{ - Task: string(content), - }) + resp := &pb.CheckTaskResponse{} + err = common.SendRequest( + ctx, + "CheckTask", + &pb.CheckTaskRequest{ + Task: string(content), + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/get_config.go b/dm/ctl/master/get_config.go index c351962395..3a2e571d56 100644 --- a/dm/ctl/master/get_config.go +++ b/dm/ctl/master/get_config.go @@ -76,14 +76,19 @@ func getCfgFunc(cmd *cobra.Command, _ []string) (err error) { return } - cli := common.MasterClient() ctx, cancel := context.WithTimeout(context.Background(), common.GlobalConfig().RPCTimeout) defer cancel() - resp, err := cli.GetCfg(ctx, &pb.GetCfgRequest{ - Type: tp, - Name: cfgName, - }) + resp := &pb.GetCfgResponse{} + err = common.SendRequest( + ctx, + "GetCfg", + &pb.GetCfgRequest{ + Type: tp, + Name: cfgName, + }, + &resp, + ) if err != nil { common.PrintLines("can not get %s config of %s", cfgType, cfgName) return diff --git a/dm/ctl/master/handle_error.go b/dm/ctl/master/handle_error.go index 577a706f38..028952141a 100644 --- a/dm/ctl/master/handle_error.go +++ b/dm/ctl/master/handle_error.go @@ -105,15 +105,21 @@ func handleErrorFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - - resp, err := cli.HandleError(ctx, &pb.HandleErrorRequest{ - Op: op, - Task: taskName, - BinlogPos: binlogPos, - Sqls: sqls, - Sources: sources, - }) + + resp := &pb.HandleErrorResponse{} + err = common.SendRequest( + ctx, + "HandleError", + &pb.HandleErrorRequest{ + Op: op, + Task: taskName, + BinlogPos: binlogPos, + Sqls: sqls, + Sources: sources, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/list_member.go b/dm/ctl/master/list_member.go index c45690e7e6..fa14af428c 100644 --- a/dm/ctl/master/list_member.go +++ b/dm/ctl/master/list_member.go @@ -80,13 +80,17 @@ func listMemberFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.ListMember(ctx, &pb.ListMemberRequest{ - Leader: leader, - Master: master, - Worker: worker, - Names: listMemberFlags.names, - }) + resp := &pb.ListMemberResponse{} + err = common.SendRequest(ctx, + "ListMember", + &pb.ListMemberRequest{ + Leader: leader, + Master: master, + Worker: worker, + Names: listMemberFlags.names, + }, + &resp, + ) if err != nil { return diff --git a/dm/ctl/master/offline_member.go b/dm/ctl/master/offline_member.go index 1757a8ece4..0aade592ac 100644 --- a/dm/ctl/master/offline_member.go +++ b/dm/ctl/master/offline_member.go @@ -81,11 +81,17 @@ func offlineMemberFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.OfflineMember(ctx, &pb.OfflineMemberRequest{ - Type: offlineType, - Name: name, - }) + resp := &pb.OfflineMemberResponse{} + err = common.SendRequest( + ctx, + "OfflineMember", + &pb.OfflineMemberRequest{ + Type: offlineType, + Name: name, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/operate_leader.go b/dm/ctl/master/operate_leader.go index 5d3356079e..9086995a59 100644 --- a/dm/ctl/master/operate_leader.go +++ b/dm/ctl/master/operate_leader.go @@ -67,10 +67,16 @@ func operateLeaderFunc(cmd *cobra.Command, _ []string) (err error) { defer cancel() // operate leader - cli := common.MasterClient() - resp, err := cli.OperateLeader(ctx, &pb.OperateLeaderRequest{ - Op: op, - }) + resp := &pb.OperateLeaderResponse{} + err = common.SendRequest( + ctx, + "OperateLeader", + &pb.OperateLeaderRequest{ + Op: op, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/operate_schema.go b/dm/ctl/master/operate_schema.go index 3f3ef70c76..49c4fd30f6 100644 --- a/dm/ctl/master/operate_schema.go +++ b/dm/ctl/master/operate_schema.go @@ -112,15 +112,21 @@ func operateSchemaCmd(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.OperateSchema(ctx, &pb.OperateSchemaRequest{ - Op: op, - Task: taskName, - Sources: sources, - Database: database, - Table: table, - Schema: string(schemaContent), - }) + resp := &pb.OperateSchemaResponse{} + err = common.SendRequest( + ctx, + "OperateSchema", + &pb.OperateSchemaRequest{ + Op: op, + Task: taskName, + Sources: sources, + Database: database, + Table: table, + Schema: string(schemaContent), + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/operate_source.go b/dm/ctl/master/operate_source.go index 8318e46701..8d55f42e66 100644 --- a/dm/ctl/master/operate_source.go +++ b/dm/ctl/master/operate_source.go @@ -115,12 +115,18 @@ func operateSourceFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.OperateSource(ctx, &pb.OperateSourceRequest{ - Config: contents, - Op: op, - SourceID: sourceID, - }) + resp := &pb.OperateSourceResponse{} + err = common.SendRequest( + ctx, + "OperateSource", + &pb.OperateSourceRequest{ + Config: contents, + Op: op, + SourceID: sourceID, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/purge_relay.go b/dm/ctl/master/purge_relay.go index 8bf9ee3d10..a5c5fe917d 100644 --- a/dm/ctl/master/purge_relay.go +++ b/dm/ctl/master/purge_relay.go @@ -99,15 +99,21 @@ func purgeRelayFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - - resp, err := cli.PurgeWorkerRelay(ctx, &pb.PurgeWorkerRelayRequest{ - Sources: sources, - //Inactive: inactive, - //Time: time2.Unix(), - Filename: filename, - SubDir: subDir, - }) + + resp := &pb.PurgeWorkerRelayResponse{} + err = common.SendRequest( + ctx, + "PurgeWorkerRelay", + &pb.PurgeWorkerRelayRequest{ + Sources: sources, + //Inactive: inactive, + //Time: time2.Unix(), + Filename: filename, + SubDir: subDir, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/query_status.go b/dm/ctl/master/query_status.go index 1c44877b2a..2ae5305ebc 100644 --- a/dm/ctl/master/query_status.go +++ b/dm/ctl/master/query_status.go @@ -65,13 +65,20 @@ func queryStatusFunc(cmd *cobra.Command, _ []string) (err error) { return } - cli := common.MasterClient() ctx, cancel := context.WithTimeout(context.Background(), common.GlobalConfig().RPCTimeout) defer cancel() - resp, err := cli.QueryStatus(ctx, &pb.QueryStatusListRequest{ - Name: taskName, - Sources: sources, - }) + + resp := &pb.QueryStatusListResponse{} + err = common.SendRequest( + ctx, + "QueryStatus", + &pb.QueryStatusListRequest{ + Name: taskName, + Sources: sources, + }, + &resp, + ) + if err != nil { common.PrintLines("can not query %s task's status(in sources %v)", taskName, sources) return diff --git a/dm/ctl/master/show_ddl_locks.go b/dm/ctl/master/show_ddl_locks.go index 07fe22cb61..c7007ddfc2 100644 --- a/dm/ctl/master/show_ddl_locks.go +++ b/dm/ctl/master/show_ddl_locks.go @@ -51,11 +51,18 @@ func showDDLLocksFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.ShowDDLLocks(ctx, &pb.ShowDDLLocksRequest{ - Task: taskName, - Sources: sources, - }) + + resp := &pb.ShowDDLLocksResponse{} + err = common.SendRequest( + ctx, + "ShowDDLLocks", + &pb.ShowDDLLocksRequest{ + Task: taskName, + Sources: sources, + }, + &resp, + ) + if err != nil { common.PrintLines("can not show DDL locks for task %s and sources %v", taskName, sources) return diff --git a/dm/ctl/master/start_task.go b/dm/ctl/master/start_task.go index e6ff151acb..5a8287fe73 100644 --- a/dm/ctl/master/start_task.go +++ b/dm/ctl/master/start_task.go @@ -64,12 +64,18 @@ func startTaskFunc(cmd *cobra.Command, _ []string) (err error) { defer cancel() // start task - cli := common.MasterClient() - resp, err := cli.StartTask(ctx, &pb.StartTaskRequest{ - Task: string(content), - Sources: sources, - RemoveMeta: removeMeta, - }) + resp := &pb.StartTaskResponse{} + err = common.SendRequest( + ctx, + "StartTask", + &pb.StartTaskRequest{ + Task: string(content), + Sources: sources, + RemoveMeta: removeMeta, + }, + &resp, + ) + if err != nil { return } diff --git a/dm/ctl/master/unlock_ddl_lock.go b/dm/ctl/master/unlock_ddl_lock.go index 2f831c36b0..7b38fc69ee 100644 --- a/dm/ctl/master/unlock_ddl_lock.go +++ b/dm/ctl/master/unlock_ddl_lock.go @@ -67,12 +67,19 @@ func unlockDDLLockFunc(cmd *cobra.Command, _ []string) (err error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cli := common.MasterClient() - resp, err := cli.UnlockDDLLock(ctx, &pb.UnlockDDLLockRequest{ - ID: lockID, - ReplaceOwner: owner, - ForceRemove: forceRemove, - }) + + resp := &pb.UnlockDDLLockResponse{} + err = common.SendRequest( + ctx, + "UnlockDDLLock", + &pb.UnlockDDLLockRequest{ + ID: lockID, + ReplaceOwner: owner, + ForceRemove: forceRemove, + }, + &resp, + ) + if err != nil { common.PrintLines("can not unlock DDL lock %s", lockID) return diff --git a/dm/ctl/master/update_task.go b/dm/ctl/master/update_task.go index 5387cd7b9e..cdecd1f4bb 100644 --- a/dm/ctl/master/update_task.go +++ b/dm/ctl/master/update_task.go @@ -57,11 +57,17 @@ func updateTaskFunc(cmd *cobra.Command, _ []string) (err error) { defer cancel() // update task - cli := common.MasterClient() - resp, err := cli.UpdateTask(ctx, &pb.UpdateTaskRequest{ - Task: string(content), - Sources: sources, - }) + resp := &pb.UpdateTaskResponse{} + err = common.SendRequest( + ctx, + "UpdateTask", + &pb.UpdateTaskRequest{ + Task: string(content), + Sources: sources, + }, + &resp, + ) + if err != nil { return } diff --git a/tests/dmctl_basic/check_list/dmctl.sh b/tests/dmctl_basic/check_list/dmctl.sh new file mode 100644 index 0000000000..550f0eb3ab --- /dev/null +++ b/tests/dmctl_basic/check_list/dmctl.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +function dmctl_multiple_addrs() { + run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT,127.0.0.1:1234" \ + "list-member" \ + "\"result\": true" 1 +} + +function dmctl_unwrap_schema() { + run_dm_ctl $WORK_DIR "http://127.0.0.1:1234,127.0.0.1:4322,https://127.0.0.1:$MASTER_PORT" \ + "list-member" \ + "\"result\": true" 1 +} + +function dmctl_wrong_addrs() { + run_dm_ctl $WORK_DIR "https://127.0.0.1:1234,127.0.0.2:1234" \ + "list-member" \ + "can't connect to https:\/\/127.0.0.1:1234,127.0.0.2:1234" 1 +} + +function dmctl_no_addr() { + $PWD/bin/dmctl.test list-member | grep 'master-addr not provided' +} diff --git a/tests/dmctl_basic/run.sh b/tests/dmctl_basic/run.sh index a5efb0b9d6..91daa71890 100755 --- a/tests/dmctl_basic/run.sh +++ b/tests/dmctl_basic/run.sh @@ -189,6 +189,12 @@ function run() { pause_task_success $TASK_NAME query_status_paused_tasks + echo "dmctl master_addr" + dmctl_multiple_addrs + dmctl_unwrap_schema + dmctl_wrong_addrs + dmctl_no_addr + # echo "update_task_worker_not_found" # update_task_worker_not_found $TASK_CONF 127.0.0.1:9999 # update_task_success_single_worker $TASK_CONF $SOURCE_ID1 diff --git a/tests/dmctl_command/run.sh b/tests/dmctl_command/run.sh index 4bdcd80b28..e8edc96bfe 100644 --- a/tests/dmctl_command/run.sh +++ b/tests/dmctl_command/run.sh @@ -9,45 +9,13 @@ WORK_DIR=$TEST_DIR/$TEST_NAME help_cnt=34 function run() { - # check dmctl alone output - # it should usage for root command - - $PWD/bin/dmctl.test DEVEL > $WORK_DIR/help.log - help_msg=$(cat $WORK_DIR/help.log) - help_msg_cnt=$(echo "${help_msg}" | wc -l |xargs) - if [ "$help_msg_cnt" != $help_cnt ]; then - echo "dmctl case 1 help failed: $help_msg" - echo $help_msg_cnt - exit 1 - fi - # check dmctl output with help flag # it should usage for root command $PWD/bin/dmctl.test DEVEL --help > $WORK_DIR/help.log help_msg=$(cat $WORK_DIR/help.log) help_msg_cnt=$(echo "${help_msg}" | wc -l |xargs) if [ "$help_msg_cnt" != $help_cnt ]; then - echo "dmctl case 2 help failed: $help_msg" - exit 1 - fi - - # check dmctl command start-task alone output - # it should usage for start-task - $PWD/bin/dmctl.test DEVEL start-task > $WORK_DIR/help.log - help_msg=$(cat $WORK_DIR/help.log) - echo $help_msg | grep -q "dmctl start-task" - if [ $? -ne 0 ]; then - echo "dmctl case 3 help failed: $help_msg" - exit 1 - fi - - # check dmctl command start-task output with help flag - # it should usage for start-task - $PWD/bin/dmctl.test DEVEL start-task --help > $WORK_DIR/help.log - help_msg=$(cat $WORK_DIR/help.log) - echo $help_msg | grep -q "dmctl start-task" - if [ $? -ne 0 ]; then - echo "dmctl case 4 help failed: $help_msg" + echo "dmctl case 1 help failed: $help_msg" exit 1 fi @@ -60,7 +28,7 @@ function run() { echo $help_msg | grep -q "dmctl start-task" if [ $? -ne 0 ]; then - echo "dmctl case 5 help failed: $help_msg" + echo "dmctl case 2 help failed: $help_msg" exit 1 fi @@ -70,12 +38,12 @@ function run() { help_msg=$(cat $WORK_DIR/help.log) help_msg_cnt=$(echo "${help_msg}" | wc -l |xargs) if [ "$help_msg_cnt" != 1 ]; then - echo "dmctl case 6 help failed: $help_msg" + echo "dmctl case 3 help failed: $help_msg" exit 1 fi echo $help_msg | grep -q "parse cmd flags err: 'xxxx' is an invalid flag" if [ $? -ne 0 ]; then - echo "dmctl case 6 help failed: $help_msg" + echo "dmctl case 3 help failed: $help_msg" exit 1 fi diff --git a/tests/ha_cases/run.sh b/tests/ha_cases/run.sh index d7c05fa060..0dbdb20a33 100755 --- a/tests/ha_cases/run.sh +++ b/tests/ha_cases/run.sh @@ -125,10 +125,7 @@ function test_join_masters_and_worker { check_port_offline $MASTER_PORT1 20 rm -rf $WORK_DIR/master1/default.master1 - # uncomment after we support https://github.com/pingcap/dm/issues/1110 - # run_dm_ctl_with_retry $WORK_DIR 127.0.0.1:$MASTER_PORT2 "list-member --worker --name=worker2" '"stage": "free",' 1 - - sleep 5 + run_dm_ctl_with_retry $WORK_DIR 127.0.0.1:$MASTER_PORT2 "list-member --worker --name=worker2" '"stage": "free",' 1 echo "join worker with 5 masters endpoint" run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker-join1.toml