From b06fa01360bf4647d2623332223a44beee56d812 Mon Sep 17 00:00:00 2001 From: 9547 Date: Tue, 22 Jun 2021 23:27:02 +0800 Subject: [PATCH 1/5] embed: rename blackbox.yml to blackbox.yml.tpl, support tls --- embed/templates/config/blackbox.yml | 38 ------------------- embed/templates/config/blackbox.yml.tpl | 49 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 38 deletions(-) delete mode 100644 embed/templates/config/blackbox.yml create mode 100644 embed/templates/config/blackbox.yml.tpl diff --git a/embed/templates/config/blackbox.yml b/embed/templates/config/blackbox.yml deleted file mode 100644 index b73f53291a..0000000000 --- a/embed/templates/config/blackbox.yml +++ /dev/null @@ -1,38 +0,0 @@ -modules: - http_2xx: - prober: http - http: - method: GET - http_post_2xx: - prober: http - http: - method: POST - tcp_connect: - prober: tcp - pop3s_banner: - prober: tcp - tcp: - query_response: - - expect: "^+OK" - tls: true - tls_config: - insecure_skip_verify: false - ssh_banner: - prober: tcp - tcp: - query_response: - - expect: "^SSH-2.0-" - irc_banner: - prober: tcp - tcp: - query_response: - - send: "NICK prober" - - send: "USER prober prober prober :prober" - - expect: "PING :([^ ]+)" - send: "PONG ${1}" - - expect: "^:[^ ]+ 001" - icmp: - prober: icmp - timeout: 5s - icmp: - preferred_ip_protocol: "ip4" diff --git a/embed/templates/config/blackbox.yml.tpl b/embed/templates/config/blackbox.yml.tpl new file mode 100644 index 0000000000..3739ad164a --- /dev/null +++ b/embed/templates/config/blackbox.yml.tpl @@ -0,0 +1,49 @@ +modules: + http_2xx: + prober: http + http: + method: GET + http_post_2xx: + prober: http + http: + method: POST + tcp_connect: + prober: tcp +{{- if .TLSEnabled}} + tls_connect: + prober: tcp + tcp: + tls: true + tls_config: + insecure_skip_verify: false + ca_file: {{.DeployDir}}/tls/ca.crt + cert_file: {{.DeployDir}}/tls/blackbox_exporter.crt + key_file: {{.DeployDir}}/tls/blackbox_exporter.pem +{{- end}} + pop3s_banner: + prober: tcp + tcp: + query_response: + - expect: '^+OK' + tls: true + tls_config: + insecure_skip_verify: false + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: '^SSH-2.0-' + irc_banner: + prober: tcp + tcp: + query_response: + - send: 'NICK prober' + - send: 'USER prober prober prober :prober' + - expect: 'PING :([^ ]+)' + send: 'PONG ${1}' + - expect: '^:[^ ]+ 001' + icmp: + prober: icmp + timeout: 5s + icmp: + preferred_ip_protocol: 'ip4' From 98798ad857b59542fad5b116103e30b1f1789426 Mon Sep 17 00:00:00 2001 From: 9547 Date: Tue, 22 Jun 2021 23:50:02 +0800 Subject: [PATCH 2/5] cluster: deploy blackbox with tls support --- pkg/cluster/manager/builder.go | 82 ++++++++++++++++++------- pkg/cluster/manager/deploy.go | 22 ++++--- pkg/cluster/task/builder.go | 10 ++- pkg/cluster/task/monitored_config.go | 17 +++-- pkg/cluster/task/tls.go | 24 ++++---- pkg/cluster/template/config/blackbox.go | 12 +++- 6 files changed, 113 insertions(+), 54 deletions(-) diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 166e0ca2ab..db02d12dab 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -212,10 +212,16 @@ func buildScaleOutTask( iterErr = err return } - tb = tb.TLSCert(inst, ca, meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) + tb = tb.TLSCert( + inst.GetHost(), + inst.ComponentName(), + inst.Role(), + inst.GetMainPort(), + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }) } t := tb.ScaleConfig(name, @@ -282,9 +288,8 @@ func buildScaleOutTask( } // Deploy monitor relevant components to remote - dlTasks, dpTasks := buildMonitoredDeployTask( - m.bindVersion, - specManager, + dlTasks, dpTasks, err := buildMonitoredDeployTask( + m, name, uninitializedHosts, topo.BaseTopo().GlobalOptions, @@ -292,6 +297,9 @@ func buildScaleOutTask( base.Version, gOpt, ) + if err != nil { + return nil, err + } downloadCompTasks = append(downloadCompTasks, convertStepDisplaysToTasks(dlTasks)...) deployCompTasks = append(deployCompTasks, convertStepDisplaysToTasks(dpTasks)...) @@ -358,15 +366,14 @@ func convertStepDisplaysToTasks(t []*task.StepDisplay) []task.Task { } func buildMonitoredDeployTask( - bindVersion spec.BindVersion, - specManager *spec.SpecManager, + m *Manager, name string, uniqueHosts map[string]hostInfo, // host -> ssh-port, os, arch globalOptions *spec.GlobalOptions, monitoredOptions *spec.MonitoredOptions, version string, gOpt operator.Options, -) (downloadCompTasks []*task.StepDisplay, deployCompTasks []*task.StepDisplay) { +) (downloadCompTasks []*task.StepDisplay, deployCompTasks []*task.StepDisplay, err error) { if monitoredOptions == nil { return } @@ -374,7 +381,7 @@ func buildMonitoredDeployTask( uniqueCompOSArch := set.NewStringSet() // monitoring agents for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { - version := bindVersion(comp, version) + version := m.bindVersion(comp, version) for host, info := range uniqueHosts { // populate unique comp-os-arch set @@ -395,8 +402,21 @@ func buildMonitoredDeployTask( } // log dir will always be with values, but might not used by the component logDir := spec.Abs(globalOptions.User, monitoredOptions.LogDir) + + deployDirs := []string{ + deployDir, + dataDir, + logDir, + filepath.Join(deployDir, "bin"), + filepath.Join(deployDir, "conf"), + filepath.Join(deployDir, "scripts"), + } + if globalOptions.TLSEnabled { + deployDirs = append(deployDirs, filepath.Join(deployDir, "tls")) + } + // Deploy component - t := task.NewBuilder(). + tb := task.NewBuilder(). UserSSH( host, info.ssh, @@ -406,11 +426,7 @@ func buildMonitoredDeployTask( gOpt.SSHType, globalOptions.SSHType, ). - Mkdir(globalOptions.User, host, - deployDir, dataDir, logDir, - filepath.Join(deployDir, "bin"), - filepath.Join(deployDir, "conf"), - filepath.Join(deployDir, "scripts")). + Mkdir(globalOptions.User, host, deployDirs...). CopyComponent( comp, info.os, @@ -427,15 +443,38 @@ func buildMonitoredDeployTask( globalOptions.ResourceControl, monitoredOptions, globalOptions.User, + globalOptions.TLSEnabled, meta.DirPaths{ Deploy: deployDir, Data: []string{dataDir}, Log: logDir, - Cache: specManager.Path(name, spec.TempConfigPath), + Cache: m.specManager.Path(name, spec.TempConfigPath), }, - ). - BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", comp, host)) - deployCompTasks = append(deployCompTasks, t) + ) + + if globalOptions.TLSEnabled && comp == spec.ComponentBlackboxExporter { + ca, innerr := crypto.ReadCA( + name, + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCACert), + m.specManager.Path(name, spec.TLSCertKeyDir, spec.TLSCAKey), + ) + if innerr != nil { + err = innerr + return + } + tb = tb.TLSCert( + host, + spec.ComponentBlackboxExporter, + spec.ComponentBlackboxExporter, + monitoredOptions.BlackboxExporterPort, + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }) + } + + deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", comp, host))) } } return @@ -485,6 +524,7 @@ func buildRefreshMonitoredConfigTasks( globalOptions.ResourceControl, monitoredOptions, globalOptions.User, + globalOptions.TLSEnabled, meta.DirPaths{ Deploy: deployDir, Data: []string{dataDir}, diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index 613c174e39..6afb538725 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -309,10 +309,16 @@ func (m *Manager) Deploy( // generate and transfer tls cert for instance if globalOptions.TLSEnabled { - t = t.TLSCert(inst, ca, meta.DirPaths{ - Deploy: deployDir, - Cache: m.specManager.Path(name, spec.TempConfigPath), - }) + t = t.TLSCert( + inst.GetHost(), + inst.ComponentName(), + inst.Role(), + inst.GetMainPort(), + ca, + meta.DirPaths{ + Deploy: deployDir, + Cache: m.specManager.Path(name, spec.TempConfigPath), + }) } // generate configs for the component @@ -341,9 +347,8 @@ func (m *Manager) Deploy( } // Deploy monitor relevant components to remote - dlTasks, dpTasks := buildMonitoredDeployTask( - m.bindVersion, - m.specManager, + dlTasks, dpTasks, err := buildMonitoredDeployTask( + m, name, uniqueHosts, globalOptions, @@ -351,6 +356,9 @@ func (m *Manager) Deploy( clusterVersion, gOpt, ) + if err != nil { + return err + } downloadCompTasks = append(downloadCompTasks, dlTasks...) deployCompTasks = append(deployCompTasks, dpTasks...) diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index bb675a6872..e91523426c 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -225,7 +225,7 @@ func (b *Builder) ScaleConfig(clusterName, clusterVersion string, specManager *s } // MonitoredConfig appends a CopyComponent task to the current task collection -func (b *Builder) MonitoredConfig(name, comp, host string, globResCtl meta.ResourceControl, options *spec.MonitoredOptions, deployUser string, paths meta.DirPaths) *Builder { +func (b *Builder) MonitoredConfig(name, comp, host string, globResCtl meta.ResourceControl, options *spec.MonitoredOptions, deployUser string, tlsEnabled bool, paths meta.DirPaths) *Builder { b.tasks = append(b.tasks, &MonitoredConfig{ name: name, component: comp, @@ -233,6 +233,7 @@ func (b *Builder) MonitoredConfig(name, comp, host string, globResCtl meta.Resou globResCtl: globResCtl, options: options, deployUser: deployUser, + tlsEnabled: tlsEnabled, paths: paths, }) return b @@ -401,10 +402,13 @@ func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployD } // TLSCert generates certificate for instance and transfers it to the server -func (b *Builder) TLSCert(inst spec.Instance, ca *crypto.CertificateAuthority, paths meta.DirPaths) *Builder { +func (b *Builder) TLSCert(host, comp, role string, port int, ca *crypto.CertificateAuthority, paths meta.DirPaths) *Builder { b.tasks = append(b.tasks, &TLSCert{ + host: host, + comp: comp, + role: role, + port: port, ca: ca, - inst: inst, paths: paths, }) return b diff --git a/pkg/cluster/task/monitored_config.go b/pkg/cluster/task/monitored_config.go index 006c593f55..076115ee03 100644 --- a/pkg/cluster/task/monitored_config.go +++ b/pkg/cluster/task/monitored_config.go @@ -40,6 +40,7 @@ type MonitoredConfig struct { globResCtl meta.ResourceControl options *spec.MonitoredOptions deployUser string + tlsEnabled bool paths meta.DirPaths } @@ -66,19 +67,17 @@ func (m *MonitoredConfig) Execute(ctx context.Context) error { var cfg template.ConfigGenerator switch m.component { case spec.ComponentNodeExporter: - if err := m.syncBlackboxConfig(ctx, exec, config.NewBlackboxConfig()); err != nil { + if err := m.syncBlackboxConfig(ctx, exec, config.NewBlackboxConfig(m.paths.Deploy, m.tlsEnabled)); err != nil { return err } - cfg = scripts.NewNodeExporterScript( - m.paths.Deploy, - m.paths.Log, - ).WithPort(uint64(m.options.NodeExporterPort)). + cfg = scripts. + NewNodeExporterScript(m.paths.Deploy, m.paths.Log). + WithPort(uint64(m.options.NodeExporterPort)). WithNumaNode(m.options.NumaNode) case spec.ComponentBlackboxExporter: - cfg = scripts.NewBlackboxExporterScript( - m.paths.Deploy, - m.paths.Log, - ).WithPort(uint64(m.options.BlackboxExporterPort)) + cfg = scripts. + NewBlackboxExporterScript(m.paths.Deploy, m.paths.Log). + WithPort(uint64(m.options.BlackboxExporterPort)) default: return fmt.Errorf("unknown monitored component %s", m.component) } diff --git a/pkg/cluster/task/tls.go b/pkg/cluster/task/tls.go index 69e52ee6d9..803d13eda5 100644 --- a/pkg/cluster/task/tls.go +++ b/pkg/cluster/task/tls.go @@ -30,7 +30,10 @@ import ( // TLSCert generates a certificate for instance type TLSCert struct { - inst spec.Instance + comp string + role string + host string + port int ca *crypto.CertificateAuthority paths meta.DirPaths } @@ -42,12 +45,12 @@ func (c *TLSCert) Execute(ctx context.Context) error { return err } - hosts := []string{c.inst.GetHost()} + hosts := []string{c.host} ips := []string{} - if net.ParseIP(c.inst.GetHost()) != nil { + if net.ParseIP(c.host) != nil { hosts, ips = ips, hosts } - csr, err := privKey.CSR(c.inst.Role(), c.inst.ComponentName(), hosts, ips) + csr, err := privKey.CSR(c.role, c.comp, hosts, ips) if err != nil { return err } @@ -62,8 +65,8 @@ func (c *TLSCert) Execute(ctx context.Context) error { } // save cert to cache dir - keyFileName := fmt.Sprintf("%s-%s-%d.pem", c.inst.Role(), c.inst.GetHost(), c.inst.GetMainPort()) - certFileName := fmt.Sprintf("%s-%s-%d.crt", c.inst.Role(), c.inst.GetHost(), c.inst.GetMainPort()) + keyFileName := fmt.Sprintf("%s-%s-%d.pem", c.role, c.host, c.port) + certFileName := fmt.Sprintf("%s-%s-%d.crt", c.role, c.host, c.port) keyFile := filepath.Join( c.paths.Cache, keyFileName, @@ -90,7 +93,7 @@ func (c *TLSCert) Execute(ctx context.Context) error { } // transfer file to remote - e, ok := ctxt.GetInner(ctx).GetExecutor(c.inst.GetHost()) + e, ok := ctxt.GetInner(ctx).GetExecutor(c.host) if !ok { return ErrNoExecutor } @@ -101,13 +104,13 @@ func (c *TLSCert) Execute(ctx context.Context) error { return errors.Annotate(err, "failed to transfer CA cert to server") } if err := e.Transfer(ctx, keyFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.pem", c.inst.Role())), + filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.pem", c.role)), false, /* download */ 0 /* limit */); err != nil { return errors.Annotate(err, "failed to transfer TLS private key to server") } if err := e.Transfer(ctx, certFile, - filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.crt", c.inst.Role())), + filepath.Join(c.paths.Deploy, "tls", fmt.Sprintf("%s.crt", c.role)), false, /* download */ 0 /* limit */); err != nil { return errors.Annotate(err, "failed to transfer TLS cert to server") @@ -123,6 +126,5 @@ func (c *TLSCert) Rollback(ctx context.Context) error { // String implements the fmt.Stringer interface func (c *TLSCert) String() string { - return fmt.Sprintf("TLSCert: host=%s role=%s cn=%s", - c.inst.GetHost(), c.inst.Role(), c.inst.ComponentName()) + return fmt.Sprintf("TLSCert: host=%s role=%s cn=%s", c.host, c.role, c.comp) } diff --git a/pkg/cluster/template/config/blackbox.go b/pkg/cluster/template/config/blackbox.go index 5773c9bfe2..599ec152b3 100644 --- a/pkg/cluster/template/config/blackbox.go +++ b/pkg/cluster/template/config/blackbox.go @@ -21,11 +21,17 @@ import ( ) // BlackboxConfig represent the data to generate AlertManager config -type BlackboxConfig struct{} +type BlackboxConfig struct { + DeployDir string + TLSEnabled bool +} // NewBlackboxConfig returns a BlackboxConfig -func NewBlackboxConfig() *BlackboxConfig { - return &BlackboxConfig{} +func NewBlackboxConfig(deployDir string, tlsEnabled bool) *BlackboxConfig { + return &BlackboxConfig{ + DeployDir: deployDir, + TLSEnabled: tlsEnabled, + } } // Config generate the config file data. From 2945f58fcc05ad5cca2d977345cc2ba6522adf6b Mon Sep 17 00:00:00 2001 From: 9547 Date: Tue, 22 Jun 2021 23:50:57 +0800 Subject: [PATCH 3/5] embed: prometheus tls_connect or tcp_connect --- embed/templates/config/prometheus.yml.tpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/embed/templates/config/prometheus.yml.tpl b/embed/templates/config/prometheus.yml.tpl index 3a5482228b..9102145779 100644 --- a/embed/templates/config/prometheus.yml.tpl +++ b/embed/templates/config/prometheus.yml.tpl @@ -210,7 +210,11 @@ scrape_configs: scrape_interval: 30s metrics_path: /probe params: +{{- if .TLSEnabled}} + module: [tls_connect] +{{- else}} module: [tcp_connect] +{{- end}} static_configs: {{- if .KafkaAddrs}} - targets: @@ -275,7 +279,11 @@ scrape_configs: scrape_interval: 30s metrics_path: /probe params: +{{- if .TLSEnabled}} + module: [tls_connect] +{{- else}} module: [tcp_connect] +{{- end}} static_configs: - targets: {{- range .TiDBStatusAddrs}} From 27facadae8d2d80745fc1be81003d86b006d6ad9 Mon Sep 17 00:00:00 2001 From: 9547 Date: Tue, 22 Jun 2021 23:56:53 +0800 Subject: [PATCH 4/5] cluster/template: blackbox.yml -> blackbox.yml.tpl --- pkg/cluster/template/config/blackbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/template/config/blackbox.go b/pkg/cluster/template/config/blackbox.go index 599ec152b3..4c9c53692e 100644 --- a/pkg/cluster/template/config/blackbox.go +++ b/pkg/cluster/template/config/blackbox.go @@ -36,7 +36,7 @@ func NewBlackboxConfig(deployDir string, tlsEnabled bool) *BlackboxConfig { // Config generate the config file data. func (c *BlackboxConfig) Config() ([]byte, error) { - fp := path.Join("templates", "config", "blackbox.yml") + fp := path.Join("templates", "config", "blackbox.yml.tpl") tpl, err := embed.ReadFile(fp) if err != nil { return nil, err From 58cb47dc2dffb3c8fe41f78735e210c3477ef6d0 Mon Sep 17 00:00:00 2001 From: 9547 Date: Wed, 23 Jun 2021 00:13:07 +0800 Subject: [PATCH 5/5] cluster/template: template Blackbox.tpl --- pkg/cluster/template/config/blackbox.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/template/config/blackbox.go b/pkg/cluster/template/config/blackbox.go index 4c9c53692e..3d5ed728d7 100644 --- a/pkg/cluster/template/config/blackbox.go +++ b/pkg/cluster/template/config/blackbox.go @@ -14,8 +14,10 @@ package config import ( + "bytes" "os" "path" + "text/template" "github.com/pingcap/tiup/embed" ) @@ -55,5 +57,15 @@ func (c *BlackboxConfig) ConfigToFile(file string) error { // ConfigWithTemplate generate the AlertManager config content by tpl func (c *BlackboxConfig) ConfigWithTemplate(tpl string) ([]byte, error) { - return []byte(tpl), nil + tmpl, err := template.New("Blackbox").Parse(tpl) + if err != nil { + return nil, err + } + + content := bytes.NewBufferString("") + if err := tmpl.Execute(content, c); err != nil { + return nil, err + } + + return content.Bytes(), nil }