Merge branch 'master' into fix-932

.github/workflows/integrate-cluster-cmd.yaml

@@ -64,15 +64,15 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
           tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
 
       - name: Upload component log
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         uses: actions/upload-artifact@v1
         with:
@@ -87,7 +87,7 @@ jobs:
           docker ps
           df -h
           free -h
-          cat /tiup-cluster/logs/*.log || true
+          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
         working-directory: ${{ env.working-directory }}

.github/workflows/integrate-cluster-scale.yaml

@@ -64,15 +64,15 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         run: |
           docker exec tiup-cluster-control bash /tiup-cluster/tests/tiup-cluster/script/pull_log.sh /tiup-cluster/logs
           ls ${{ env.working-directory }}
           tar czvf ${{ env.working-directory }}/logs.tar.gz ${{ env.working-directory }}/logs/
 
       - name: Upload component log
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         uses: actions/upload-artifact@v1
         with:
@@ -81,14 +81,14 @@ jobs:
 
       - name: Output cluster debug log
         working-directory: ${{ env.working-directory }}
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         run: |
           pwd
           docker ps
           df -h
           free -h
-          cat /tiup-cluster/logs/*.log || true
+          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
         working-directory: ${{ env.working-directory }}

.github/workflows/integrate-dm.yaml

@@ -68,31 +68,31 @@ jobs:
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        # if: steps.test.outputs.exit_code != 0
+        # if: ${{ failure() }}
         if: always()
         run: |
           docker exec tiup-cluster-control bash -c 'mkdir -p /tiup-cluster/logs; [[ -d ~/.tiup/logs ]] && find ~/.tiup/logs -type f -name "*.log" -exec cp {} /tiup-cluster/logs \; || true'
           ls ${{ env.working-directory }}
           tar czvf ${{ env.working-directory }}/dm.logs.tar.gz ${{ env.working-directory }}/logs/
 
       - name: Upload component log
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         uses: actions/upload-artifact@v1
         with:
           name: dm_logs
           path: ${{ env.working-directory }}/dm.logs.tar.gz
 
-      - name: Output cluster debug log
+      - name: Output dm debug log
         working-directory: ${{ env.working-directory }}
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         run: |
           pwd
           docker ps
           df -h
           free -h
-          cat /tiup-cluster/logs/*.log || true
+          find ${{ env.working-directory }}/logs -type f -exec sh -c 'echo "{}" && cat {} && echo "\n"' \; || true
 
       - name: Upload coverage to Codecov
         working-directory: ${{ env.working-directory }}

.github/workflows/integrate-playground.yaml

@@ -46,23 +46,31 @@ jobs:
         working-directory: ${{ env.working-directory }}
         run: |
           export PATH=$PATH:${{ env.working-directory }}/bin/
-          bash ${{ env.working-directory }}/tests/tiup-playground/${{matrix.cases}}.sh
+          bash ${{ env.working-directory }}/tests/tiup-playground/${{ matrix.cases }}.sh
 
       - name: Collect component log
         working-directory: ${{ env.working-directory }}
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         run: |
-          find ${{ env.working-directory }}/tests/tiup-playground/_tmp/home/data -type f | grep -E '*/ti.*/ti.*.log$' | xargs tar czvf ${{ env.working-directory }}/playground.logs.tar.gz
+          # ignore the rocksdb
+          find ${{ env.working-directory }}/tests/tiup-playground/_tmp/home/data -type f -name "*.log" | grep -vE '/data/(raft|db|region-meta)/' | xargs tar czvf ${{ env.working-directory }}/playground.logs.tar.gz
 
       - name: Upload component log
-        if: steps.test.outputs.exit_code != 0
+        if: ${{ failure() }}
         # if: always()
         uses: actions/upload-artifact@v1
         with:
           name: playground_logs
           path: ${{ env.working-directory }}/playground.logs.tar.gz
 
+      - name: Output playground debug log
+        working-directory: ${{ env.working-directory }}
+        if: ${{ failure() }}
+        # if: always()
+        run: |
+          for f in $(find ${{ env.working-directory }}/tests/tiup-playground/_tmp/home/data -type f -name "*.log" | grep -vE '/data/(raft|db|region-meta)/'); do echo "${f}" && cat "${f}"; done
+
       - name: Upload coverage to Codecov
         working-directory: ${{ env.working-directory }}
         run: |

CHANGELOG.md

@@ -1,5 +1,19 @@
 TiUP Changelog
 
+## [1.2.5] 2020.11.27
+
+### Fixes
+
+- Fix the issue that can't operate the cluster which have tispark workers without tispark master ([#924](https://github.com/pingcap/tiup/pull/924), [@AstroProfundis](https://github.com/AstroProfundis))
+  - Root cause: once the tispark master been removed from the cluster, any later action will be reject by TiUP
+  - Fix: make it possible for broken clusters to fix no tispark master error by scaling out a new tispark master node
+- Fix the issue that it report `pump node id not found` while drainer node id not found ([#925](https://github.com/pingcap/tiup/pull/925), [@lucklove](https://github.com/lucklove))
+
+### Improvements
+
+- Support deploy TiFlash on multi-disks with "storage" configurations since v4.0.9 ([#931](https://github.com/pingcap/tiup/pull/931), [#938](https://github.com/pingcap/tiup/pull/938), [@JaySon-Huang](https://github.com/JaySon-Huang))
+- Check duplicated pd_servers.name in the topology before truly deploy the cluster ([#922](https://github.com/pingcap/tiup/pull/922), [@anywhy](https://github.com/anywhy))
+
 ## [1.2.4] 2020.11.19
 
 ### Fixes

components/dm/command/scale_in.go

@@ -80,11 +80,13 @@ func ScaleInDMCluster(
 ) error {
 	// instances by uuid
 	instances := map[string]dm.Instance{}
+	instCount := map[string]int{}
 
 	// make sure all nodeIds exists in topology
 	for _, component := range topo.ComponentsByStartOrder() {
 		for _, instance := range component.Instances() {
 			instances[instance.ID()] = instance
+			instCount[instance.GetHost()]++
 		}
 	}
 
@@ -110,7 +112,8 @@ func ScaleInDMCluster(
 				if !deletedNodes.Exist(instance.ID()) {
 					continue
 				}
-				if err := operator.StopAndDestroyInstance(getter, topo, instance, options, false); err != nil {
+				instCount[instance.GetHost()]--
+				if err := operator.StopAndDestroyInstance(getter, topo, instance, options, instCount[instance.GetHost()] == 0); err != nil {
 					log.Warnf("failed to stop/destroy %s: %v", component.Name(), err)
 				}
 			}
@@ -162,6 +165,14 @@ func ScaleInDMCluster(
 			if err := operator.DestroyComponent(getter, []dm.Instance{instance}, topo, options); err != nil {
 				return errors.Annotatef(err, "failed to destroy %s", component.Name())
 			}
+
+			instCount[instance.GetHost()]--
+			if instCount[instance.GetHost()] == 0 {
+				if err := operator.DeletePublicKey(getter, instance.GetHost()); err != nil {
+					return errors.Annotatef(err, "failed to delete public key")
+				}
+			}
+
 		}
 	}
 

components/dm/spec/logic.go

@@ -28,8 +28,8 @@ import (
 
 // Components names supported by TiOps
 const (
-	ComponentDMMaster     = "dm-master"
-	ComponentDMWorker     = "dm-worker"
+	ComponentDMMaster     = spec.ComponentDMMaster
+	ComponentDMWorker     = spec.ComponentDMWorker
 	ComponentPrometheus   = spec.ComponentPrometheus
 	ComponentGrafana      = spec.ComponentGrafana
 	ComponentAlertmanager = spec.ComponentAlertmanager

components/dm/spec/topology_dm.go

@@ -557,6 +557,11 @@ func (topo *Specification) Validate() error {
 	return spec.RelativePathDetect(topo, isSkipField)
 }
 
+// Type implements Topology interface.
+func (topo *Specification) Type() string {
+	return spec.TopoTypeDM
+}
+
 // BaseTopo implements Topology interface.
 func (topo *Specification) BaseTopo() *spec.BaseTopo {
 	return &spec.BaseTopo{

examples/topology.example.yaml

@@ -418,8 +418,10 @@ server_configs:
     #      - key: "dc"
     #        value: "sha"
   tiflash:
-    # path_realtime_mode: false
     logger.level: "info"
+    ## Deprecated multi-disks storage path setting style since v4.0.9,
+    ## check storage.* configurations in host level for new style.
+    # path_realtime_mode: false
   tiflash-learner:
     log-level: "info"
     # raftstore.apply-pool-size: 4
@@ -488,37 +490,40 @@ tiflash_servers:
     # flash_proxy_status_port: 20292
     # metrics_port: 8234
     # deploy_dir: /tidb-deploy/tiflash-9000
+    ## With cluster version >= v4.0.9 and you want to deploy a multi-disk TiFlash node, it is recommended to
+    ## check config.storage.* for details. The data_dir will be ignored if you defined those configurations.
+    ## Setting data_dir to a ','-joined string is still supported but deprecated.
+    ## Check https://docs.pingcap.com/tidb/stable/tiflash-configuration#multi-disk-deployment for more details.
     # data_dir: /tidb-data/tiflash-9000
     # log_dir: /tidb-deploy/tiflash-9000/log
     # numa_node: "0,1"
-    # #
-    # # `path_realtime_mode` only works if `data_dir` is specified with multiple paths.
-    # #
-    # #  path_realtime_mode:
-    # #    "true" means only other paths instead of first path can be used to store older data.
-    # #    "false" means all paths can be used to store older data.
-    # #
-    # # TiFlash only uses the first path to store the latest data (i.e. "delta"). And others for the older data (i.e. "stable". which is the majority of data),
-    # #
-    # # E.g, if you intend to use an fast and smaller NVMe SSD (256GB) to speed up data ingestion speed in TiFlash,
-    # # and 4 extra normal SSDs (512GB each) for main storage. Then your configurations should be look like:
-    # #
-    # # data_dir: /nvme_ssd_256/data,/ssd1_512/data,/ssd2_512/data,/ssd3_512/data,/ssd4_512/data
-    # # config:
-    # #   path_realtime_mode: true
-    # #
-    # #
-    # # And if your first disk is big enough, to fully use the capacity of it, use configurations look like:
-    # #
-    # # data_dir: /nvme_ssd_512/data,/ssd1_512/data,/ssd2_512/data,/ssd3_512/data,/ssd4_512/data
-    # # config:
-    # #   path_realtime_mode: false
-    # #
-    # #
     # # The following configs are used to overwrite the `server_configs.tiflash` values.
     # config:
-    #   path_realtime_mode: false
     #   logger.level: "info"
+    #   ## Deprecated multi-disks storage path setting. Deprecated since v4.0.9,
+    #   ## check following storage.* configurations for new style.
+    #   # path_realtime_mode: false
+    #   #
+    #   ## Multi-disks storage paths settings. These will overwrite the `data_dir`.
+    #   ## If there are multiple SSD disks on the machine,
+    #   ## specify the path list on `storage.main.dir` can improve TiFlash performance.
+    #   # storage.main.dir: [ "/nvme_ssd0_512/tiflash", "/nvme_ssd1_512/tiflash" ]
+    #   ## Store capacity of each path, i.e. max data size allowed.
+    #   ## If it is not set, or is set to 0s, the actual disk capacity is used.
+    #   # storage.main.capacity = [ 536870912000, 536870912000 ]
+    #   #
+    #   ## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
+    #   ## on the machine, 
+    #   ## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
+    #   ## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
+    #   ## can improve TiFlash performance.
+    #   # storage.main.dir: [ "/hdd0_512/tiflash", "/hdd1_512/tiflash", "/hdd2_512/tiflash" ]
+    #   # storage.latest.dir: [ "/nvme_ssd0_100/tiflash" ]
+    #   ## Store capacity of each path, i.e. max data size allowed.
+    #   ## If it is not set, or is set to 0s, the actual disk capacity is used.
+    #   # storage.main.capacity = [ 536870912000, 536870912000 ]
+    #   # storage.latest.capacity = [ 107374182400 ]
+    #
     # # The following configs are used to overwrite the `server_configs.tiflash-learner` values.
     # learner_config:
     #   log-level: "info"

pkg/cluster/executor/executor.go

@@ -47,6 +47,9 @@ var (
 	// It's used to predict if the connection can establish success in the future.
 	// Its main purpose is to avoid sshpass hang when user speficied a wrong prompt.
 	connectionTestCommand = "echo connection test, if killed, check the password prompt"
+
+	// SSH authorized_keys file
+	defaultSSHAuthorizedKeys = "~/.ssh/authorized_keys"
 )
 
 // Executor is the executor interface for TiOps, all tasks will in the end
@@ -155,3 +158,30 @@ func checkLocalIP(ip string) error {
 
 	return fmt.Errorf("address %s not found in all interfaces, found ips: %s", ip, strings.Join(foundIps, ","))
 }
+
+// FindSSHAuthorizedKeysFile finds the correct path of SSH authorized keys file
+func FindSSHAuthorizedKeysFile(exec Executor) string {
+	// detect if custom path of authorized keys file is set
+	// NOTE: we do not yet support:
+	//   - custom config for user (~/.ssh/config)
+	//   - sshd started with custom config (other than /etc/ssh/sshd_config)
+	//   - ssh server implementations other than OpenSSH (such as dropbear)
+	sshAuthorizedKeys := defaultSSHAuthorizedKeys
+	cmd := "grep -Ev '^\\s*#|^\\s*$' /etc/ssh/sshd_config"
+	stdout, _, _ := exec.Execute(cmd, true) // error ignored as we have default value
+	for _, line := range strings.Split(string(stdout), "\n") {
+		if !strings.Contains(line, "AuthorizedKeysFile") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) >= 2 {
+			sshAuthorizedKeys = fields[1]
+			break
+		}
+	}
+
+	if !strings.HasPrefix(sshAuthorizedKeys, "/") && !strings.HasPrefix(sshAuthorizedKeys, "~") {
+		sshAuthorizedKeys = fmt.Sprintf("~/%s", sshAuthorizedKeys)
+	}
+	return sshAuthorizedKeys
+}

pkg/cluster/operation/action.go

@@ -114,7 +114,7 @@ func Stop(
 
 	instCount := map[string]int{}
 	cluster.IterInstance(func(inst spec.Instance) {
-		instCount[inst.GetHost()] = instCount[inst.GetHost()] + 1
+		instCount[inst.GetHost()]++
 	})
 
 	for _, comp := range components {