Merge branch 'master' into what_the_help

CHANGELOG.md

@@ -1,5 +1,12 @@
 TiUP Changelog
 
+## [1.9.4] 2022-04-12
+
+## Fixes
+
+- Fix copy error when file is read only in `tiup-playground` ([#1816](https://github.com/pingcap/tiup/pull/1816), [@breeswish](https://github.com/breeswish))
+- Fix `data-dir` not properly handled for TiCDC v6.0.0 in `tiup-cluster` ([#1838](https://github.com/pingcap/tiup/pull/1838), [@overvenus](https://github.com/overvenus))
+
 ## [1.9.3] 2022-03-24
 
 ### Fixes

cmd/history.go

@@ -0,0 +1,107 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/pingcap/errors"
+	"github.com/pingcap/tiup/pkg/environment"
+	"github.com/pingcap/tiup/pkg/tui"
+	"github.com/spf13/cobra"
+)
+
+// newHistoryCmd  history
+func newHistoryCmd() *cobra.Command {
+	rows := 100
+	var displayMode string
+	var all bool
+	cmd := &cobra.Command{
+		Use:   "history <rows>",
+		Short: "Display the historical execution record of TiUP, displays 100 lines by default",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) > 0 {
+				r, err := strconv.Atoi(args[0])
+				if err == nil {
+					rows = r
+				} else {
+					return fmt.Errorf("%s: numeric argument required", args[0])
+				}
+			}
+
+			env := environment.GlobalEnv()
+			rows, err := env.GetHistory(rows, all)
+			if err != nil {
+				return err
+			}
+
+			if displayMode == "json" {
+				for _, r := range rows {
+					rBytes, err := json.Marshal(r)
+					if err != nil {
+						continue
+					}
+					fmt.Println(string(rBytes))
+				}
+				return nil
+			}
+			var table [][]string
+			table = append(table, []string{"Date", "Command", "Code"})
+
+			for _, r := range rows {
+				table = append(table, []string{
+					r.Date.Format("2006-01-02T15:04:05"),
+					r.Command,
+					strconv.Itoa(r.Code),
+				})
+			}
+			tui.PrintTable(table, true)
+			fmt.Printf("history log save path: %s\n", env.LocalPath(environment.HistoryDir))
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&displayMode, "format", "default", "The format of output, available values are [default, json]")
+	cmd.Flags().BoolVar(&all, "all", false, "Display all execution history")
+	cmd.AddCommand(newHistoryCleanupCmd())
+	return cmd
+}
+
+func newHistoryCleanupCmd() *cobra.Command {
+	var retainDays int
+	var all bool
+	var skipConfirm bool
+	cmd := &cobra.Command{
+		Use:   "cleanup",
+		Short: "delete all execution history",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if retainDays < 0 {
+				return errors.Errorf("retain-days cannot be less than 0")
+			}
+
+			if all {
+				retainDays = 0
+			}
+
+			env := environment.GlobalEnv()
+			return env.DeleteHistory(retainDays, skipConfirm)
+		},
+	}
+
+	cmd.Flags().IntVar(&retainDays, "retain-days", 60, "Number of days to keep history for deletion")
+	cmd.Flags().BoolVar(&all, "all", false, "Delete all history")
+	cmd.Flags().BoolVarP(&skipConfirm, "yes", "y", false, "Skip all confirmations and assumes 'yes'")
+	return cmd
+}

cmd/root.go

@@ -215,6 +215,7 @@ the latest stable version will be downloaded from the repository.`,
 		newMirrorCmd(),
 		newTelemetryCmd(),
 		newEnvCmd(),
+		newHistoryCmd(),
 	)
 }
 
@@ -247,6 +248,12 @@ func Execute() {
 		// us a dedicated package for that
 		reportEnabled = false
 	} else {
+		// record TiUP execution history
+		err := environment.HistoryRecord(env, os.Args, start, code)
+		if err != nil {
+			log.Warnf("Record TiUP execution history log failed: %v", err)
+		}
+
 		teleMeta, _, err := telemetry.GetMeta(env)
 		if err == nil {
 			reportEnabled = teleMeta.Status == telemetry.EnableStatus

components/cluster/command/meta.go

@@ -0,0 +1,65 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+func newMetaCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "meta",
+		Short: "backup/restore meta information",
+	}
+
+	var filePath string
+
+	var metaBackupCmd = &cobra.Command{
+		Use:   "backup <cluster-name>",
+		Short: "backup topology and other information of cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return fmt.Errorf("please input cluster-name")
+			}
+			if filePath == "" {
+				filePath = "tiup-cluster_" + args[0] + "_metabackup_" + time.Now().Format(time.RFC3339) + ".tar.gz"
+			}
+			err := cm.BackupClusterMeta(args[0], filePath)
+			if err == nil {
+				log.Infof("successfully backup meta of cluster %s on %s", args[0], filePath)
+			}
+			return err
+		},
+	}
+	metaBackupCmd.Flags().StringVar(&filePath, "file", "", "filepath of output tarball")
+
+	var metaRestoreCmd = &cobra.Command{
+		Use:   "restore <cluster-name> <backup-file>",
+		Short: "restore topology and other information of cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 2 {
+				return fmt.Errorf("please input cluster-name and backup-file")
+			}
+			return cm.RestoreClusterMeta(args[0], args[1], skipConfirm)
+		},
+	}
+
+	cmd.AddCommand(metaBackupCmd)
+	cmd.AddCommand(metaRestoreCmd)
+
+	return cmd
+}

components/cluster/command/root.go

@@ -213,6 +213,7 @@ func init() {
 		newReplayCmd(),
 		newTemplateCmd(),
 		newTLSCmd(),
+		newMetaCmd(),
 	)
 }
 

components/dm/command/meta.go

@@ -0,0 +1,65 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+func newMetaCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "meta",
+		Short: "backup/restore meta information",
+	}
+
+	var filePath string
+
+	var metaBackupCmd = &cobra.Command{
+		Use:   "backup <cluster-name>",
+		Short: "backup topology and other information of a cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				return fmt.Errorf("please input cluster name")
+			}
+			if filePath == "" {
+				filePath = "tiup-cluster_" + args[0] + "_metabackup_" + time.Now().Format(time.RFC3339) + ".tar.gz"
+			}
+			err := cm.BackupClusterMeta(args[0], filePath)
+			if err == nil {
+				log.Infof("successfully backup meta of cluster %s on %s", args[0], filePath)
+			}
+			return err
+		},
+	}
+	metaBackupCmd.Flags().StringVar(&filePath, "file", "", "filepath of output tarball")
+
+	var metaRestoreCmd = &cobra.Command{
+		Use:   "restore <cluster-name> <backup-file>",
+		Short: "restore topology and other information of a cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 2 {
+				return fmt.Errorf("please input cluster name and path to the backup file")
+			}
+			return cm.RestoreClusterMeta(args[0], args[1], skipConfirm)
+		},
+	}
+
+	cmd.AddCommand(metaBackupCmd)
+	cmd.AddCommand(metaRestoreCmd)
+
+	return cmd
+}

components/dm/command/root.go

@@ -169,6 +169,7 @@ please backup your data before process.`,
 		newDisableCmd(),
 		newReplayCmd(),
 		newTemplateCmd(),
+		newMetaCmd(),
 	)
 }
 

embed/examples/cluster/topology.example.yaml

@@ -50,7 +50,7 @@ monitored:
 # # All configuration items use points to represent the hierarchy, e.g:
 # #   readpool.storage.use-unified-pool
 # #           ^       ^
-# # - example: https://github.com/pingcap/tiup/blob/master/embed/templates/examples/topology.example.yaml.
+# # - example: https://github.com/pingcap/tiup/blob/master/embed/examples/cluster/topology.example.yaml
 # # You can overwrite this configuration via the instance-level `config` field.
 # server_configs:
   # tidb:
@@ -294,7 +294,8 @@ monitoring_servers:
     # data_dir: "/tidb-data/prometheus-8249"
     # # Prometheus log file storage directory.
     # log_dir: "/tidb-deploy/prometheus-8249/log"
-
+    # prometheus rule dir on TiUP machine
+    # rule_dir: /home/tidb/prometheus_rule
 # # Server configs are used to specify the configuration of Grafana Servers.  
 grafana_servers:
   # # The ip address of the Grafana Server.
@@ -303,6 +304,10 @@ grafana_servers:
     # port: 3000
     # # Grafana deployment file, startup script, configuration file storage directory.
     # deploy_dir: /tidb-deploy/grafana-3000
+    # grafana dashboard dir on TiUP machine
+    # dashboard_dir: /home/tidb/dashboards
+    # config:
+    #   log.file.level: warning
 
 # # Server configs are used to specify the configuration of Alertmanager Servers.  
 alertmanager_servers:

embed/examples/dm/topology.example.yaml

@@ -59,11 +59,15 @@ monitoring_servers:
     # deploy_dir: "/tidb-deploy/prometheus-8249"
     # data_dir: "/tidb-data/prometheus-8249"
     # log_dir: "/tidb-deploy/prometheus-8249/log"
+    # prometheus rule dir on TiUP machine
+    # rule_dir: /home/tidb/prometheus_rule
 
 grafana_servers:
   - host: 10.0.1.14
     # port: 3000
     # deploy_dir: /tidb-deploy/grafana-3000
+    # grafana dashboard dir on TiUP machine
+    # dashboard_dir: /home/tidb/dashboards
     # config:       # Enable this part if you want to use WebUI, make sure tiup dm -v newer than v1.9.0.
     #   auth.anonymous.enabled: true
     #   security.allow_embedding: true

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20220125073028-58f2ac94aa38
 	github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect
 	github.com/pingcap/tidb-insight/collector v0.0.0-20220111101533-227008e9835b
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.32.1
 	github.com/prometheus/prom2json v1.3.0