*: refactor logging to support custom settings for different session (#…

…1651)
pingcap · Dec 7, 2021 · f873e74 · f873e74
1 parent ff08d40
commit f873e74
Show file tree

Hide file tree

Showing 94 changed files with 1,043 additions and 659 deletions.
diff --git a/cmd/mirror.go b/cmd/mirror.go
@@ -32,7 +32,6 @@ import (
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/localdata"
-	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/repository"
 	"github.com/pingcap/tiup/pkg/repository/model"
 	ru "github.com/pingcap/tiup/pkg/repository/utils"

diff --git a/cmd/root.go b/cmd/root.go
@@ -27,7 +27,7 @@ import (
 	"github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/exec"
 	"github.com/pingcap/tiup/pkg/localdata"
-	"github.com/pingcap/tiup/pkg/logger/log"
+	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/pingcap/tiup/pkg/repository"
 	"github.com/pingcap/tiup/pkg/repository/v1manifest"
 	"github.com/pingcap/tiup/pkg/telemetry"
@@ -41,6 +41,7 @@ var (
 	reportEnabled bool // is telemetry report enabled
 	eventUUID     = uuid.New().String()
 	teleCommand   string
+	log           = logprinter.NewLogger("") // use default logger
 )
 
 // arguments

diff --git a/components/cluster/command/deploy.go b/components/cluster/command/deploy.go
@@ -80,7 +80,7 @@ func newDeploy() *cobra.Command {
 }
 
 func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) {
-	nodeInfoTask := task.NewBuilder(gOpt.DisplayMode).Func("Check status", func(ctx context.Context) error {
+	nodeInfoTask := task.NewBuilder(builder.Logger).Func("Check status", func(ctx context.Context) error {
 		var err error
 		teleNodeInfos, err = operator.GetNodeInfo(ctx, topo)
 		_ = err
@@ -92,7 +92,7 @@ func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Opt
 		builder.ParallelStep("+ Check status", false, nodeInfoTask)
 	}
 
-	enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error {
+	enableTask := task.NewBuilder(builder.Logger).Func("Setting service auto start on boot", func(ctx context.Context) error {
 		return operator.Enable(ctx, topo, operator.Options{}, true)
 	}).BuildAsStep("Enable service").SetHidden(true)
 

diff --git a/components/cluster/command/display.go b/components/cluster/command/display.go
@@ -14,17 +14,11 @@
 package command
 
 import (
-	"crypto/tls"
 	"errors"
 	"fmt"
-	"net/url"
-	"time"
 
-	"github.com/fatih/color"
 	perrs "github.com/pingcap/errors"
-	"github.com/pingcap/tiup/pkg/cluster/api"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
-	"github.com/pingcap/tiup/pkg/crypto"
 	"github.com/pingcap/tiup/pkg/meta"
 	"github.com/spf13/cobra"
 )
@@ -73,7 +67,7 @@ func newDisplayCmd() *cobra.Command {
 				if err != nil {
 					return err
 				}
-				return displayDashboardInfo(clusterName, tlsCfg)
+				return cm.DisplayDashboardInfo(clusterName, tlsCfg)
 			}
 			if showTiKVLabels {
 				return cm.DisplayTiKVLabels(clusterName, gOpt)
@@ -91,51 +85,3 @@ func newDisplayCmd() *cobra.Command {
 
 	return cmd
 }
-
-func displayDashboardInfo(clusterName string, tlsCfg *tls.Config) error {
-	metadata, err := spec.ClusterMetadata(clusterName)
-	if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) &&
-		!errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) {
-		return err
-	}
-
-	pdEndpoints := make([]string, 0)
-	for _, pd := range metadata.Topology.PDServers {
-		pdEndpoints = append(pdEndpoints, fmt.Sprintf("%s:%d", pd.Host, pd.ClientPort))
-	}
-
-	pdAPI := api.NewPDClient(pdEndpoints, 2*time.Second, tlsCfg)
-	dashboardAddr, err := pdAPI.GetDashboardAddress()
-	if err != nil {
-		return fmt.Errorf("failed to retrieve TiDB Dashboard instance from PD: %s", err)
-	}
-	if dashboardAddr == "auto" {
-		return fmt.Errorf("TiDB Dashboard is not initialized, please start PD and try again")
-	} else if dashboardAddr == "none" {
-		return fmt.Errorf("TiDB Dashboard is disabled")
-	}
-
-	u, err := url.Parse(dashboardAddr)
-	if err != nil {
-		return fmt.Errorf("unknown TiDB Dashboard PD instance: %s", dashboardAddr)
-	}
-
-	u.Path = "/dashboard/"
-
-	if tlsCfg != nil {
-		fmt.Println(
-			"Client certificate:",
-			color.CyanString(tidbSpec.Path(clusterName, spec.TLSCertKeyDir, spec.PFXClientCert)),
-		)
-		fmt.Println(
-			"Certificate password:",
-			color.CyanString(crypto.PKCS12Password),
-		)
-	}
-	fmt.Println(
-		"Dashboard URL:",
-		color.CyanString(u.String()),
-	)
-
-	return nil
-}
diff --git a/components/cluster/command/import.go b/components/cluster/command/import.go
@@ -14,14 +14,15 @@
 package command
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/pingcap/tiup/pkg/cluster/ansible"
+	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
-	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/tui"
 	tiuputils "github.com/pingcap/tiup/pkg/utils"
 	"github.com/spf13/cobra"
@@ -49,8 +50,14 @@ func newImportCmd() *cobra.Command {
 				ansibleDir = cwd
 			}
 
+			ctx := ctxt.New(
+				context.Background(),
+				gOpt.Concurrency,
+				log,
+			)
+
 			// migrate cluster metadata from Ansible inventory
-			clsName, clsMeta, inv, err := ansible.ReadInventory(ansibleDir, inventoryFileName)
+			clsName, clsMeta, inv, err := ansible.ReadInventory(ctx, ansibleDir, inventoryFileName)
 			if err != nil {
 				return err
 			}
@@ -103,7 +110,15 @@ func newImportCmd() *cobra.Command {
 			}
 
 			// parse config and import nodes
-			if err = ansible.ParseAndImportInventory(ansibleDir, ansibleCfgFile, clsMeta, inv, gOpt.SSHTimeout, gOpt.SSHType); err != nil {
+			if err = ansible.ParseAndImportInventory(
+				ctx,
+				ansibleDir,
+				ansibleCfgFile,
+				clsMeta,
+				inv,
+				gOpt.SSHTimeout,
+				gOpt.SSHType,
+			); err != nil {
 				return err
 			}
 
@@ -123,7 +138,7 @@ func newImportCmd() *cobra.Command {
 			}
 
 			// copy config files form deployment servers
-			if err = ansible.ImportConfig(clsName, clsMeta, gOpt); err != nil {
+			if err = ansible.ImportConfig(ctx, clsName, clsMeta, gOpt); err != nil {
 				return err
 			}
 

diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go
@@ -34,7 +34,7 @@ import (
 	tiupmeta "github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/localdata"
 	"github.com/pingcap/tiup/pkg/logger"
-	"github.com/pingcap/tiup/pkg/logger/log"
+	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/pingcap/tiup/pkg/proxy"
 	"github.com/pingcap/tiup/pkg/repository"
 	"github.com/pingcap/tiup/pkg/telemetry"
@@ -56,6 +56,7 @@ var (
 	teleNodeInfos []*telemetry.NodeInfo
 	teleTopology  string
 	teleCommand   []string
+	log           = logprinter.NewLogger("") // init default logger
 )
 
 var tidbSpec *spec.SpecManager
@@ -100,14 +101,8 @@ func init() {
 		SilenceErrors: true,
 		Version:       version.NewTiUPVersion().String(),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			switch strings.ToLower(gOpt.DisplayMode) {
-			case "json":
-				log.SetDisplayMode(log.DisplayModeJSON)
-			case "plain", "text":
-				log.SetDisplayMode(log.DisplayModePlain)
-			default:
-				log.SetDisplayMode(log.DisplayModeDefault)
-			}
+			// populate logger
+			log.SetDisplayModeFromString(gOpt.DisplayMode)
 
 			var err error
 			var env *tiupmeta.Environment
@@ -116,7 +111,7 @@ func init() {
 			}
 
 			tidbSpec = spec.GetSpecManager()
-			cm = manager.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion)
+			cm = manager.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion, log)
 			logger.EnableAuditLog(spec.AuditDir())
 
 			// Running in other OS/ARCH Should be fine we only download manifest file.
@@ -140,7 +135,14 @@ func init() {
 				log.Infof("The --native-ssh flag has been deprecated, please use --ssh=system")
 			}
 
-			err = proxy.MaybeStartProxy(gOpt.SSHProxyHost, gOpt.SSHProxyPort, gOpt.SSHProxyUser, gOpt.SSHProxyUsePassword, gOpt.SSHProxyIdentity)
+			err = proxy.MaybeStartProxy(
+				gOpt.SSHProxyHost,
+				gOpt.SSHProxyPort,
+				gOpt.SSHProxyUser,
+				gOpt.SSHProxyUsePassword,
+				gOpt.SSHProxyIdentity,
+				log,
+			)
 			if err != nil {
 				return perrs.Annotate(err, "start http-proxy")
 			}
@@ -349,7 +351,7 @@ func Execute() {
 	}
 
 	switch log.GetDisplayMode() {
-	case log.DisplayModeJSON:
+	case logprinter.DisplayModeJSON:
 		obj := struct {
 			Code int    `json:"exit_code"`
 			Err  string `json:"error,omitempty"`

diff --git a/components/dm/ansible/import_test.go b/components/dm/ansible/import_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/tiup/components/dm/spec"
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/executor"
+	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/stretchr/testify/require"
 )
 
@@ -142,7 +143,11 @@ func TestImportFromAnsible(t *testing.T) {
 	im, err := NewImporter(dir, "inventory.ini", executor.SSHTypeBuiltin, 0)
 	assert.Nil(err)
 	im.testExecutorGetter = &executorGetter{}
-	clusterName, meta, err := im.ImportFromAnsibleDir(ctxt.New(context.Background(), 0))
+	clusterName, meta, err := im.ImportFromAnsibleDir(ctxt.New(
+		context.Background(),
+		0,
+		logprinter.NewLogger(""),
+	))
 	assert.Nil(err, "verbose: %+v", err)
 	assert.Equal("test-cluster", clusterName)
 

diff --git a/components/dm/command/deploy.go b/components/dm/command/deploy.go
@@ -82,7 +82,7 @@ func supportVersion(vs string) error {
 }
 
 func postDeployHook(builder *task.Builder, topo spec.Topology, gOpt operator.Options) {
-	enableTask := task.NewBuilder(gOpt.DisplayMode).Func("Setting service auto start on boot", func(ctx context.Context) error {
+	enableTask := task.NewBuilder(builder.Logger).Func("Setting service auto start on boot", func(ctx context.Context) error {
 		return operator.Enable(ctx, topo, operator.Options{}, true)
 	}).BuildAsStep("Enable service").SetHidden(true)
 

diff --git a/components/dm/command/import.go b/components/dm/command/import.go
@@ -49,7 +49,7 @@ func newImportCmd() *cobra.Command {
 				return err
 			}
 
-			ctx := ctxt.New(context.Background(), 0)
+			ctx := ctxt.New(context.Background(), 0, log)
 			clusterName, meta, err := importer.ImportFromAnsibleDir(ctx)
 			if err != nil {
 				return err

diff --git a/components/dm/command/root.go b/components/dm/command/root.go
@@ -31,6 +31,7 @@ import (
 	tiupmeta "github.com/pingcap/tiup/pkg/environment"
 	"github.com/pingcap/tiup/pkg/localdata"
 	"github.com/pingcap/tiup/pkg/logger"
+	logprinter "github.com/pingcap/tiup/pkg/logger/printer"
 	"github.com/pingcap/tiup/pkg/proxy"
 	"github.com/pingcap/tiup/pkg/repository"
 	"github.com/pingcap/tiup/pkg/tui"
@@ -46,6 +47,7 @@ var (
 	rootCmd     *cobra.Command
 	gOpt        operator.Options
 	skipConfirm bool
+	log         = logprinter.NewLogger("") // init default logger
 )
 
 var dmspec *cspec.SpecManager
@@ -72,6 +74,9 @@ please backup your data before process.`,
 		SilenceErrors: true,
 		Version:       version.NewTiUPVersion().String(),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			// populate logger
+			log.SetDisplayModeFromString(gOpt.DisplayMode)
+
 			var err error
 			var env *tiupmeta.Environment
 			if err = cspec.Initialize("dm"); err != nil {
@@ -80,7 +85,7 @@ please backup your data before process.`,
 
 			dmspec = spec.GetSpecManager()
 			logger.EnableAuditLog(cspec.AuditDir())
-			cm = manager.NewManager("dm", dmspec, spec.DMComponentVersion)
+			cm = manager.NewManager("dm", dmspec, spec.DMComponentVersion, log)
 
 			// Running in other OS/ARCH Should be fine we only download manifest file.
 			env, err = tiupmeta.InitEnv(repository.Options{
@@ -99,7 +104,14 @@ please backup your data before process.`,
 				fmt.Println("The --native-ssh flag has been deprecated, please use --ssh=system")
 			}
 
-			err = proxy.MaybeStartProxy(gOpt.SSHProxyHost, gOpt.SSHProxyPort, gOpt.SSHProxyUser, gOpt.SSHProxyUsePassword, gOpt.SSHProxyIdentity)
+			err = proxy.MaybeStartProxy(
+				gOpt.SSHProxyHost,
+				gOpt.SSHProxyPort,
+				gOpt.SSHProxyUser,
+				gOpt.SSHProxyUsePassword,
+				gOpt.SSHProxyIdentity,
+				log,
+			)
 			if err != nil {
 				return perrs.Annotate(err, "start http-proxy")
 			}

diff --git a/components/dm/command/scale_in.go b/components/dm/command/scale_in.go
@@ -26,7 +26,6 @@ import (
 	operator "github.com/pingcap/tiup/pkg/cluster/operation"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/task"
-	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/set"
 	"github.com/spf13/cobra"
 )

diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go
@@ -21,12 +21,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/pingcap/tiup/pkg/logger/log"
-	"github.com/pingcap/tiup/pkg/meta"
-
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
+	"github.com/pingcap/tiup/pkg/meta"
 )
 
 // Components names supported by TiUP
@@ -91,7 +89,7 @@ func (c *DMMasterComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: s.Status,
-				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+				UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration {
 					return spec.UptimeByHost(s.Host, s.Port, tlsCfg)
 				},
 			},
@@ -171,7 +169,6 @@ func (i *MasterInstance) ScaleConfig(
 	).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(c.Endpoints(deployUser)...)
 
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort()))
-	log.Infof("script path: %s", fp)
 	if err := cfg.ConfigToFile(fp); err != nil {
 		return err
 	}
@@ -222,7 +219,7 @@ func (c *DMWorkerComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: s.Status,
-				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+				UptimeFn: func(_ context.Context, tlsCfg *tls.Config) time.Duration {
 					return spec.UptimeByHost(s.Host, s.Port, tlsCfg)
 				},
 			},