add test for get-config

chaos/cases/task.go

@@ -23,13 +23,15 @@ import (
 
  "github.com/chaos-mesh/go-sqlsmith"
  "github.com/pingcap/errors"
+ "github.com/pingcap/parser/mysql"
  "go.uber.org/zap"
  "golang.org/x/sync/errgroup"
 
  config2 "github.com/pingcap/dm/dm/config"
  "github.com/pingcap/dm/dm/pb"
  "github.com/pingcap/dm/pkg/conn"
  "github.com/pingcap/dm/pkg/log"
+ "github.com/pingcap/dm/pkg/utils"
 )
 
 const (
@@ -363,6 +365,10 @@ func (t *task) genIncrData(ctx context.Context) (err error) {
  })
  }
  if err = eg.Wait(); err != nil {
+ if utils.IsMySQLError(err, mysql.ErrDupFieldName) {
+ t.logger.Warn("ignore duplicate field name for ddl", log.ShortError(err))
+ continue
+ }
  return err
  }
  }

dm/config/task.go

@@ -616,6 +616,7 @@ func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConf
 }
 
 func getGenerateName(rule interface{}, nameIdx int, namePrefix string, nameMap map[string]string) (string, int) {
+ // use json as key since no DeepEqual for rules now.
  ruleByte, err := json.Marshal(rule)
  if err != nil {
  log.L().Error(fmt.Sprintf("marshal %s rule to json", namePrefix), log.ShortError(err))

dm/master/server.go

@@ -1890,7 +1890,7 @@ func (s *Server) GetCfg(ctx context.Context, req *pb.GetCfgRequest) (*pb.GetCfgR
  resp2.Msg = "source not found"
  return resp2, nil
  }
- sourceCfg.From.Password = "*******"
+ sourceCfg.From.Password = "******"
  cfg, err2 = sourceCfg.Yaml()
  if err2 != nil {
  resp2.Msg = err2.Error()

dm/master/server_test.go

@@ -1567,6 +1567,7 @@ func (t *testMaster) TestGetCfg(c *check.C) {
  }
  server.scheduler, _ = testMockScheduler(ctx, &wg, c, sources, workers, "",
  makeWorkerClientsForHandle(ctrl, taskName, sources, workers, req))
+ server.etcdClient = etcdTestCli
 
  // start task
  mock := t.initVersionDB(c)
@@ -1582,6 +1583,7 @@ func (t *testMaster) TestGetCfg(c *check.C) {
  // get task config
  req1 := &pb.GetCfgRequest{
  Name: taskName,
+ Type: pb.CfgType_TaskType,
  }
  resp1, err := server.GetCfg(context.Background(), req1)
  c.Assert(err, check.IsNil)
@@ -1591,12 +1593,14 @@ func (t *testMaster) TestGetCfg(c *check.C) {
  // wrong task name
  req2 := &pb.GetCfgRequest{
  Name: "haha",
+ Type: pb.CfgType_TaskType,
  }
  resp2, err := server.GetCfg(context.Background(), req2)
  c.Assert(err, check.IsNil)
  c.Assert(resp2.Result, check.IsFalse)
+ c.Assert(resp2.Msg, check.Equals, "task not found")
 
- // test recover from etcd
+ // test restart master
  server.scheduler.Close()
  c.Assert(server.scheduler.Start(ctx, etcdTestCli), check.IsNil)
 
@@ -1605,6 +1609,51 @@ func (t *testMaster) TestGetCfg(c *check.C) {
  c.Assert(resp3.Result, check.IsTrue)
  c.Assert(resp3.Cfg, check.Equals, resp1.Cfg)
 
+ req3 := &pb.GetCfgRequest{
+ Name: "dm-master",
+ Type: pb.CfgType_MasterType,
+ }
+ resp4, err := server.GetCfg(context.Background(), req3)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp4.Result, check.IsTrue)
+ c.Assert(strings.Contains(resp4.Cfg, "name = \"dm-master\""), check.IsTrue)
+
+ req4 := &pb.GetCfgRequest{
+ Name: "haha",
+ Type: pb.CfgType_MasterType,
+ }
+ resp5, err := server.GetCfg(context.Background(), req4)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp5.Result, check.IsFalse)
+ c.Assert(resp5.Msg, check.Equals, "master not found")
+
+ req5 := &pb.GetCfgRequest{
+ Name: "haha",
+ Type: pb.CfgType_WorkerType,
+ }
+ resp6, err := server.GetCfg(context.Background(), req5)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp6.Result, check.IsFalse)
+ c.Assert(resp6.Msg, check.Equals, "worker not found")
+
+ req6 := &pb.GetCfgRequest{
+ Name: "mysql-replica-01",
+ Type: pb.CfgType_SourceType,
+ }
+ resp7, err := server.GetCfg(context.Background(), req6)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp7.Result, check.IsTrue)
+ c.Assert(strings.Contains(resp7.Cfg, "source-id: mysql-replica-01"), check.IsTrue, check.Commentf(resp7.Cfg))
+
+ req7 := &pb.GetCfgRequest{
+ Name: "haha",
+ Type: pb.CfgType_SourceType,
+ }
+ resp8, err := server.GetCfg(context.Background(), req7)
+ c.Assert(err, check.IsNil)
+ c.Assert(resp8.Result, check.IsFalse)
+ c.Assert(resp8.Msg, check.Equals, "source not found")
+
  clearSchedulerEnv(c, cancel, &wg)
 }
 

dm/worker/relay.go

@@ -227,11 +227,12 @@ func (h *realRelayHolder) resumeRelay(ctx context.Context, op pb.RelayOp) error
 
 func (h *realRelayHolder) stopRelay(ctx context.Context, op pb.RelayOp) error {
  h.Lock()
- defer h.Unlock()
  if h.stage == pb.Stage_Stopped {
+ h.Unlock()
  return terror.ErrWorkerRelayStageNotValid.Generatef("current stage is already stopped not valid, relayop %s", op)
  }
  h.stage = pb.Stage_Stopped
+ h.Unlock() // unlock to make `run` can return
 
  // now, when try to stop relay unit, we close relay holder
  h.Close()

tests/dmctl_basic/check_list/get_config.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+function get_config_wrong_arg() {
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config" \
+ "get-config <task\/master\/worker\/source> <name> \[--file filename\] \[flags\]" 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config hihi haha" \
+ "invalid config type 'hihi'" 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config task haha" \
+ "task not found" 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config master haha" \
+ "master not found" 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config worker haha" \
+ "worker not found" 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config source haha" \
+ "source not found" 1
+}
+
+function diff_get_config() {
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config task test --file $WORK_DIR/get_task.yaml" \
+ "\"result\": true" 1
+ diff $WORK_DIR/get_task.yaml $cur/conf/get_task.yaml || exit 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config master master1 --file $dm_master_conf" \
+ "\"result\": true" 1
+ diff $dm_master_conf $cur/conf/get_master1.toml || exit 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config worker worker1 --file $dm_worker1_conf" \
+ "\"result\": true" 1
+ diff $dm_worker1_conf $cur/conf/get_worker1.toml || exit 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config worker worker2 --file $dm_worker2_conf" \
+ "\"result\": true" 1
+ diff $dm_worker2_conf $cur/conf/get_worker2.toml || exit 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config source mysql-replica-01 --file $WORK_DIR/get_source1.yaml" \
+ "\"result\": true" 1
+ diff $WORK_DIR/get_source1.yaml $cur/conf/get_source1.yaml || exit 1
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "get-config source mysql-replica-02 --file $WORK_DIR/get_source2.yaml" \
+ "\"result\": true" 1
+ diff $WORK_DIR/get_source2.yaml $cur/conf/get_source2.yaml || exit 1
+}
+
+function get_config_to_file() {
+ diff_get_config
+
+ sed -i "s/password: '\*\*\*\*\*\*'/password: ''/g" $WORK_DIR/get_task.yaml
+ sed -i "s/password: '\*\*\*\*\*\*'/password: '123456'/g" $WORK_DIR/get_source1.yaml
+ sed -i "s/password: '\*\*\*\*\*\*'/password: '123456'/g" $WORK_DIR/get_source2.yaml
+
+ # stop task
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "stop-task test" \
+ "\"result\": true" 3
+
+ # restart source with get config
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "operate-source stop mysql-replica-01" \
+ "\"result\": true" 2
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "operate-source create $WORK_DIR/get_source1.yaml" \
+ "\"result\": true" 2
+
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "operate-source stop mysql-replica-02" \
+ "\"result\": true" 2
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "operate-source create $WORK_DIR/get_source2.yaml" \
+ "\"result\": true" 2
+
+ # start task with get config
+ run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "start-task $WORK_DIR/get_task.yaml" \
+ "\"result\": true" 3
+
+ # restart master with get config
+ ps aux | grep dm-master |awk '{print $2}'|xargs kill || true
+ check_port_offline $MASTER_PORT1 20
+ run_dm_master $WORK_DIR/master $MASTER_PORT $dm_master_conf
+ check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT
+
+ # restart worker with get config
+ ps aux | grep worker1 |awk '{print $2}'|xargs kill || true
+ check_port_offline $WORKER1_PORT 20
+ ps aux | grep worker2 |awk '{print $2}'|xargs kill || true
+ check_port_offline $WORKER2_PORT 20
+ run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "list-member --worker" \
+ "offline" 2
+
+ run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $dm_worker1_conf
+ check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
+ run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $dm_worker2_conf
+ check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
+ run_dm_ctl_with_retry $WORK_DIR "127.0.0.1:$MASTER_PORT" \
+ "list-member --worker" \
+ "bound" 2
+
+ diff_get_config
+}

tests/dmctl_basic/conf/dm-master.toml

@@ -1,4 +1,5 @@
 # Master Configuration.
 
+name = "master1"
 master-addr = ":8261"
 advertise-addr = "127.0.0.1:8261"

tests/dmctl_basic/conf/get_master1.toml

@@ -0,0 +1,21 @@
+log-level = "debug"
+log-file = "/tmp/dm_test/dmctl_basic/master/log/dm-master.log"
+log-format = "text"
+log-rotate = ""
+rpc-timeout = "30s"
+rpc-rate-limit = 10.0
+rpc-rate-burst = 40
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+config-file = "/tmp/dm_test/dmctl_basic/master/dm-master.toml"
+name = "master1"
+data-dir = "default.master1"
+peer-urls = "http://127.0.0.1:8291"
+advertise-peer-urls = "http://127.0.0.1:8291"
+initial-cluster = "master1=http://127.0.0.1:8291"
+initial-cluster-state = "new"
+join = ""
+v1-sources-path = ""
+ssl-ca = ""
+ssl-cert = ""
+ssl-key = ""

tests/dmctl_basic/conf/get_source1.yaml

@@ -0,0 +1,32 @@
+enable-gtid: false
+auto-fix-gtid: false
+relay-dir: /tmp/dm_test/dmctl_basic/worker1/relay_log
+meta-dir: ""
+flavor: mysql
+charset: ""
+enable-relay: true
+relay-binlog-name: ""
+relay-binlog-gtid: ""
+source-id: mysql-replica-01
+from:
+ host: 127.0.0.1
+ port: 3306
+ user: root
+ password: '******'
+ max-allowed-packet: null
+ session: {}
+ security: null
+purge:
+ interval: 3600
+ expires: 0
+ remain-space: 15
+checker:
+ check-enable: true
+ backoff-rollback: 5m0s
+ backoff-max: 5m0s
+ check-interval: 5s
+ backoff-min: 1s
+ backoff-jitter: true
+ backoff-factor: 2
+server-id: 123456
+tracer: {}