diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index 9aead4e806..56a86edfe2 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -46,6 +46,7 @@ spec: {{- end }} - -pd-failover-period={{ .Values.controllerManager.pdFailoverPeriod | default "5m" }} - -tikv-failover-period={{ .Values.controllerManager.tikvFailoverPeriod | default "5m" }} + - -tiflash-failover-period={{ .Values.controllerManager.tiflashFailoverPeriod | default "5m" }} - -tidb-failover-period={{ .Values.controllerManager.tidbFailoverPeriod | default "5m" }} - -v={{ .Values.controllerManager.logLevel }} {{- if .Values.testMode }} diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index 30f1c8951e..f9e25006dc 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -58,6 +58,8 @@ controllerManager: tikvFailoverPeriod: 5m # tidb failover period default(5m) tidbFailoverPeriod: 5m + # tiflash failover period default(5m) + tiflashFailoverPeriod: 5m ## affinity defines pod scheduling rules,affinity default settings is empty. ## please read the affinity document before set your scheduling rule: ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index aca09196a9..a5ae55c7a3 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -52,16 +52,17 @@ import ( ) var ( - printVersion bool - workers int - autoFailover bool - pdFailoverPeriod time.Duration - tikvFailoverPeriod time.Duration - tidbFailoverPeriod time.Duration - leaseDuration = 15 * time.Second - renewDuration = 5 * time.Second - retryPeriod = 3 * time.Second - waitDuration = 5 * time.Second + printVersion bool + workers int + autoFailover bool + pdFailoverPeriod time.Duration + tikvFailoverPeriod time.Duration + tidbFailoverPeriod time.Duration + tiflashFailoverPeriod time.Duration + leaseDuration = 15 * time.Second + renewDuration = 5 * time.Second + retryPeriod = 3 * time.Second + waitDuration = 5 * time.Second ) func init() { @@ -72,6 +73,7 @@ func init() { flag.BoolVar(&autoFailover, "auto-failover", true, "Auto failover") flag.DurationVar(&pdFailoverPeriod, "pd-failover-period", time.Duration(5*time.Minute), "PD failover period default(5m)") flag.DurationVar(&tikvFailoverPeriod, "tikv-failover-period", time.Duration(5*time.Minute), "TiKV failover period default(5m)") + flag.DurationVar(&tiflashFailoverPeriod, "tiflash-failover-period", time.Duration(5*time.Minute), "TiFlash failover period default(5m)") flag.DurationVar(&tidbFailoverPeriod, "tidb-failover-period", time.Duration(5*time.Minute), "TiDB failover period") flag.DurationVar(&controller.ResyncDuration, "resync-duration", time.Duration(30*time.Second), "Resync time of informer") flag.BoolVar(&controller.TestMode, "test-mode", false, "whether tidb-operator run in test mode") @@ -179,7 +181,7 @@ func main() { klog.Fatalf("failed to upgrade: %v", err) } - tcController := tidbcluster.NewController(kubeCli, cli, genericCli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod) + tcController := tidbcluster.NewController(kubeCli, cli, genericCli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod, tiflashFailoverPeriod) backupController := backup.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) restoreController := restore.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) bsController := backupschedule.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) diff --git a/docs/api-references/config.json b/docs/api-references/config.json index cc214e90bd..f0888cb044 100644 --- a/docs/api-references/config.json +++ b/docs/api-references/config.json @@ -9,12 +9,13 @@ "TCPPort", "HTTPPort", "InternalServerHTTPPort", - "Errorlog", + "ErrorLog", "ServerLog", "TiDBStatusAddr", "ServiceAddr", "ProxyConfig", "ClusterManagerPath", + "Flash", "FlashStatus", "FlashQuota", "FlashUser", @@ -27,7 +28,18 @@ "hideTypePatterns": [ "ParseError$", "List$", - "DataResource" + "DataResource", + "ProxyConfig", + "^Flash$", + "FlashCluster", + "FlashStatus", + "FlashQuota", + "FlashUser", + "FlashProfile", + "FlashApplication", + "FlashProxy", + "FlashServerConfig", + "FlashRaft" ], "externalPackages": [ { diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index e2ee9cfbc8..189656a5fa 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -2489,19 +2489,6 @@ int64 -flash
- - -Flash - - - - -(Optional) - - - - loger
@@ -3207,92 +3194,6 @@ FlashCluster -

FlashApplication -

-

-(Appears on: -CommonConfig) -

-

-

FlashApplication is the configuration of [application] section.

-

- - - - - - - - - - - - - -
FieldDescription
-runAsDaemon
- -bool - -		 -(Optional) -

Optional: Defaults to true

-
-

FlashCluster -

-

-(Appears on: -Flash) -

-

-

FlashCluster is the configuration of [flash.flash_cluster] section.

-

- - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-refresh_interval
- -int32 - -		 -(Optional) -

Optional: Defaults to 20

-
-update_rule_interval
- -int32 - -		 -(Optional) -

Optional: Defaults to 10

-
-master_ttl
- -int32 - -		 -(Optional) -

Optional: Defaults to 60

-

FlashLogger

@@ -3348,335 +3249,6 @@ int32 -

FlashProfile -

-

-(Appears on: -CommonConfig) -

-

-

FlashProfile is the configuration of [profiles] section.

-

- - - - - - - - - - - - - - - - - -
FieldDescription
-readonly
- - -Profile - - -		 -(Optional) -
-default
- - -Profile - - -		 -(Optional) -
-

FlashProxy -

-

-(Appears on: -Flash) -

-

-

FlashProxy is the configuration of [flash.proxy] section.

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-addr
- -string - -		 -(Optional) -

Optional: Defaults to 0.0.0.0:20170

-
-advertise-addr
- -string - -		 -(Optional) -
-data-dir
- -string - -		 -(Optional) -

Optional: Defaults to /data/proxy

-
-config
- -string - -		 -(Optional) -

Optional: Defaults to /data/proxy.toml

-
-log-file
- -string - -		 -(Optional) -

Optional: Defaults to /data/logs/proxy.log

-
-

FlashQuota -

-

-(Appears on: -CommonConfig) -

-

-

FlashQuota is the configuration of [quotas] section.

-

- - - - - - - - - - - - - -
FieldDescription
-default
- - -Quota - - -		 -(Optional) -
-

FlashRaft -

-

-(Appears on: -CommonConfig) -

-

-

FlashRaft is the configuration of [raft] section.

-

- - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-pd_addr
- -string - -		 -(Optional) -
-kvstore_path
- -string - -		 -(Optional) -

Optional: Defaults to /data/kvstore

-
-storage_engine
- -string - -		 -(Optional) -

Optional: Defaults to dt

-
-

FlashServerConfig -

-

-(Appears on: -ProxyConfig) -

-

-

FlashServerConfig is the configuration of Proxy server.

-

- - - - - - - - - - - - - - - - - -
FieldDescription
-engine-addr
- -string - -		 -(Optional) -
-TiKVServerConfig
- - -TiKVServerConfig - - -		 -

-(Members of TiKVServerConfig are embedded into this type.) -

-
-

FlashStatus -

-

-(Appears on: -CommonConfig) -

-

-

FlashStatus is the configuration of [status] section.

-

- - - - - - - - - - - - - -
FieldDescription
-metrics_port
- -int32 - -		 -(Optional) -

Optional: Defaults to 8234

-
-

FlashUser -

-

-(Appears on: -CommonConfig) -

-

-

FlashUser is the configuration of [users] section.

-

- - - - - - - - - - - - - - - - - -
FieldDescription
-readonly
- - -User - - -		 -(Optional) -
-default
- - -User - - -		 -

GcsStorageProvider

@@ -4250,6 +3822,40 @@ uint32 +

LogTailerSpec +

+

+(Appears on: +TiFlashSpec) +

+

+

LogTailerSpec represents an optional log tailer sidecar container

+

+ + + + + + + + + + + + + +
FieldDescription
+ResourceRequirements
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +

+(Members of ResourceRequirements are embedded into this type.) +

+

MasterKeyFileConfig

@@ -6721,10 +6327,6 @@ float64

Profile

-(Appears on: -FlashProfile) -

-

Profile is the configuration profiles.

@@ -6848,215 +6450,6 @@ int
-

ProxyConfig -

-

-(Appears on: -TiFlashConfig) -

-

-

ProxyConfig is the configuration of TiFlash proxy process. -All the configurations are same with those of TiKV except adding engine-addr in the TiKVServerConfig

-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-log-level
- -string - -		 -(Optional) -

Optional: Defaults to info

-
-log-file
- -string - -		 -(Optional) -
-log-rotation-timespan
- -string - -		 -(Optional) -

Optional: Defaults to 24h

-
-panic-when-unexpected-key-or-data
- -bool - -		 -(Optional) -
-server
- - -FlashServerConfig - - -		 -(Optional) -
-storage
- - -TiKVStorageConfig - - -		 -(Optional) -
-raftstore
- - -TiKVRaftstoreConfig - - -		 -(Optional) -
-rocksdb
- - -TiKVDbConfig - - -		 -(Optional) -
-coprocessor
- - -TiKVCoprocessorConfig - - -		 -(Optional) -
-readpool
- - -TiKVReadPoolConfig - - -		 -(Optional) -
-raftdb
- - -TiKVRaftDBConfig - - -		 -(Optional) -
-import
- - -TiKVImportConfig - - -		 -(Optional) -
-gc
- - -TiKVGCConfig - - -		 -(Optional) -
-pd
- - -TiKVPDConfig - - -		 -(Optional) -
-security
- - -TiKVSecurityConfig - - -		 -(Optional) -

ProxyProtocol

@@ -7102,32 +6495,6 @@ uint -

ProxyServer -

-

-

ProxyServer is the configuration of TiFlash proxy server.

-

- - - - - - - - - - - - - -
FieldDescription
-engine-addr
- -string - -		 -(Optional) -

PumpSpec

@@ -7287,10 +6654,6 @@ Kubernetes apps/v1.StatefulSetStatus

Quota

-(Appears on: -FlashQuota) -

-

Quota is the configuration of [quotas.default] section.

@@ -9599,6 +8962,20 @@ TiFlashConfig

Config is the Configuration of TiFlash

+ + + +
+logTailer
+ + +LogTailerSpec + + +		 +(Optional) +

LogTailer is the configurations of the log tailers for TiFlash

+

TiKVBlockCacheConfig @@ -10488,7 +9865,6 @@ TiKVEncryptionConfig

(Appears on: -ProxyConfig, TiKVConfig)

@@ -10698,7 +10074,6 @@ string

(Appears on: -ProxyConfig, TiKVConfig)

@@ -11180,7 +10555,6 @@ Kubernetes meta/v1.Time

(Appears on: -ProxyConfig, TiKVConfig)

@@ -11222,7 +10596,6 @@ string

(Appears on: -ProxyConfig, TiKVConfig)

@@ -11405,7 +10778,6 @@ If the type set to kms, this config should be filled

(Appears on: -ProxyConfig, TiKVConfig)

@@ -11480,7 +10852,6 @@ Optional: Defaults to 10

(Appears on: -ProxyConfig, TiKVConfig)

@@ -11765,7 +11136,6 @@ TiKVCfConfig

(Appears on: -ProxyConfig, TiKVConfig)

@@ -12356,7 +11726,6 @@ bool

(Appears on: -ProxyConfig, TiKVConfig)

@@ -12401,7 +11770,6 @@ TiKVStorageReadPoolConfig

(Appears on: -ProxyConfig, TiKVConfig)

@@ -12487,7 +11855,6 @@ string

(Appears on: -FlashServerConfig, TiKVConfig)

@@ -13011,7 +12378,6 @@ string

(Appears on: -ProxyConfig, TiKVConfig)

@@ -14052,6 +13418,18 @@ PumpStatus + + +tiflash
+ + +TiFlashStatus + + + + + +

TidbInitializerSpec @@ -14593,10 +13971,6 @@ Kubernetes meta/v1.Time

User

-(Appears on: -FlashUser) -

-

User is the configuration of users.

diff --git a/manifests/crd.yaml b/manifests/crd.yaml index b4eb49031e..cdb4d96c7a 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -5215,6 +5215,21 @@ spec: description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object + logTailer: + description: LogTailerSpec represents an optional log tailer sidecar + container + properties: + limits: + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object maxFailoverCount: description: 'MaxFailoverCount limit the max replicas could be added in failover, 0 means no failover Optional: Defaults to 3' diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 6dc513a64e..18c0663c61 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -47,6 +47,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.HelperSpec": schema_pkg_apis_pingcap_v1alpha1_HelperSpec(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.IsolationRead": schema_pkg_apis_pingcap_v1alpha1_IsolationRead(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.Log": schema_pkg_apis_pingcap_v1alpha1_Log(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.LogTailerSpec": schema_pkg_apis_pingcap_v1alpha1_LogTailerSpec(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MasterKeyFileConfig": schema_pkg_apis_pingcap_v1alpha1_MasterKeyFileConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MasterKeyKMSConfig": schema_pkg_apis_pingcap_v1alpha1_MasterKeyKMSConfig(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.MonitorContainer": schema_pkg_apis_pingcap_v1alpha1_MonitorContainer(ref), @@ -1486,6 +1487,49 @@ func schema_pkg_apis_pingcap_v1alpha1_Log(ref common.ReferenceCallback) common.O } } +func schema_pkg_apis_pingcap_v1alpha1_LogTailerSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "LogTailerSpec represents an optional log tailer sidecar container", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "limits": { + SchemaProps: spec.SchemaProps{ + Description: "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + "requests": { + SchemaProps: spec.SchemaProps{ + Description: "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + func schema_pkg_apis_pingcap_v1alpha1_MasterKeyFileConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -4416,12 +4460,18 @@ func schema_pkg_apis_pingcap_v1alpha1_TiFlashSpec(ref common.ReferenceCallback) Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiFlashConfig"), }, }, + "logTailer": { + SchemaProps: spec.SchemaProps{ + Description: "LogTailer is the configurations of the log tailers for TiFlash", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.LogTailerSpec"), + }, + }, }, Required: []string{"replicas", "storageClaims"}, }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageClaim", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiFlashConfig", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.LogTailerSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.StorageClaim", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiFlashConfig", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.PodSecurityContext", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go index 35829bf6ab..828d332481 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap/v1alpha1/tidbcluster.go @@ -74,6 +74,28 @@ func (tc *TidbCluster) TiKVContainerPrivilege() *bool { return tc.Spec.TiKV.Privileged } +func (tc *TidbCluster) TiFlashImage() string { + image := tc.Spec.TiFlash.Image + baseImage := tc.Spec.TiFlash.BaseImage + // base image takes higher priority + if baseImage != "" { + version := tc.Spec.TiFlash.Version + if version == nil { + version = &tc.Spec.Version + } + image = fmt.Sprintf("%s:%s", baseImage, *version) + } + return image +} + +func (tc *TidbCluster) TiFlashContainerPrivilege() *bool { + if tc.Spec.TiFlash.Privileged == nil { + pri := false + return &pri + } + return tc.Spec.TiFlash.Privileged +} + func (tc *TidbCluster) TiDBImage() string { image := tc.Spec.TiDB.Image baseImage := tc.Spec.TiDB.BaseImage @@ -224,6 +246,36 @@ func (tc *TidbCluster) TiKVStsActualReplicas() int32 { return stsStatus.Replicas } +func (tc *TidbCluster) TiFlashAllPodsStarted() bool { + return tc.TiFlashStsDesiredReplicas() == tc.TiFlashStsActualReplicas() +} + +func (tc *TidbCluster) TiFlashAllStoresReady() bool { + if int(tc.TiFlashStsDesiredReplicas()) != len(tc.Status.TiFlash.Stores) { + return false + } + + for _, store := range tc.Status.TiFlash.Stores { + if store.State != TiKVStateUp { + return false + } + } + + return true +} + +func (tc *TidbCluster) TiFlashStsDesiredReplicas() int32 { + return tc.Spec.TiFlash.Replicas + int32(len(tc.Status.TiFlash.FailureStores)) +} + +func (tc *TidbCluster) TiFlashStsActualReplicas() int32 { + stsStatus := tc.Status.TiFlash.StatefulSet + if stsStatus == nil { + return 0 + } + return stsStatus.Replicas +} + func (tc *TidbCluster) TiDBAllPodsStarted() bool { return tc.TiDBStsDesiredReplicas() == tc.TiDBStsActualReplicas() } diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster_component.go b/pkg/apis/pingcap/v1alpha1/tidbcluster_component.go index f8d6d62637..840c24e756 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbcluster_component.go +++ b/pkg/apis/pingcap/v1alpha1/tidbcluster_component.go @@ -175,6 +175,11 @@ func (tc *TidbCluster) BaseTiKVSpec() ComponentAccessor { return &componentAccessorImpl{&tc.Spec, &tc.Spec.TiKV.ComponentSpec} } +// BaseTiFlashSpec returns the base spec of TiFlash servers +func (tc *TidbCluster) BaseTiFlashSpec() ComponentAccessor { + return &componentAccessorImpl{&tc.Spec, &tc.Spec.TiFlash.ComponentSpec} +} + // BasePDSpec returns the base spec of PD servers func (tc *TidbCluster) BasePDSpec() ComponentAccessor { return &componentAccessorImpl{&tc.Spec, &tc.Spec.PD.ComponentSpec} diff --git a/pkg/apis/pingcap/v1alpha1/tiflash_config.go b/pkg/apis/pingcap/v1alpha1/tiflash_config.go index 8aa70999dc..be35c1da41 100644 --- a/pkg/apis/pingcap/v1alpha1/tiflash_config.go +++ b/pkg/apis/pingcap/v1alpha1/tiflash_config.go @@ -1,4 +1,4 @@ -// Copyright 2019 PingCAP, Inc. +// Copyright 2020 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -74,17 +74,10 @@ type ProxyConfig struct { Security *TiKVSecurityConfig `json:"security,omitempty" toml:"security,omitempty"` } -// ProxyServer is the configuration of TiFlash proxy server. -// +k8s:openapi-gen=false -type ProxyServer struct { - // +optional - EngineAddr string `json:"engine-addr,omitempty" toml:"engine-addr,omitempty"` -} - // CommonConfig is the configuration of TiFlash process. // +k8s:openapi-gen=true type CommonConfig struct { - // Optional: Defaults to "/data/tmp" + // Optional: Defaults to "/data0/tmp" // +optional // +k8s:openapi-gen=false TmpPath string `json:"tmp_path,omitempty" toml:"tmp_path,omitempty"` @@ -99,7 +92,7 @@ type CommonConfig struct { // +k8s:openapi-gen=false DefaultProfile string `json:"default_profile,omitempty" toml:"default_profile,omitempty"` - // Optional: Defaults to "/data/db" + // Optional: Defaults to "/data0/db" // +optional // +k8s:openapi-gen=false Path string `json:"path,omitempty" toml:"path,omitempty"` @@ -191,7 +184,7 @@ type FlashUser struct { // +k8s:openapi-gen=false type User struct { // +optional - Password string `json:"password,omitempty" toml:"password,omitempty"` + Password string `json:"password,omitempty" toml:"password"` // +optional Profile string `json:"profile,omitempty" toml:"profile,omitempty"` // +optional @@ -257,7 +250,7 @@ type FlashStatus struct { type FlashRaft struct { // +optional PDAddr string `json:"pd_addr,omitempty" toml:"pd_addr,omitempty"` - // Optional: Defaults to /data/kvstore + // Optional: Defaults to /data0/kvstore // +optional KVStorePath string `json:"kvstore_path,omitempty" toml:"kvstore_path,omitempty"` // Optional: Defaults to dt @@ -276,14 +269,14 @@ type FlashApplication struct { // FlashLogger is the configuration of [logger] section. // +k8s:openapi-gen=true type FlashLogger struct { - // Optional: Defaults to /data/logs/error.log + // Optional: Defaults to /data0/logs/error.log // +optional // +k8s:openapi-gen=false - Errorlog string `json:"errorlog,omitempty" toml:"errorlog,omitempty"` + ErrorLog string `json:"errorlog,omitempty" toml:"errorlog,omitempty"` // Optional: Defaults to 100M // +optional Size string `json:"size,omitempty" toml:"size,omitempty"` - // Optional: Defaults to /data/logs/server.log + // Optional: Defaults to /data0/logs/server.log // +optional // +k8s:openapi-gen=false ServerLog string `json:"log,omitempty" toml:"log,omitempty"` @@ -324,7 +317,7 @@ type FlashCluster struct { // +optional // +k8s:openapi-gen=false ClusterManagerPath string `json:"cluster_manager_path,omitempty" toml:"cluster_manager_path,omitempty"` - // Optional: Defaults to /data/logs/flash_cluster_manager.log + // Optional: Defaults to /data0/logs/flash_cluster_manager.log // +optional // +k8s:openapi-gen=false ClusterLog string `json:"log,omitempty" toml:"log,omitempty"` @@ -347,13 +340,13 @@ type FlashProxy struct { Addr string `json:"addr,omitempty" toml:"addr,omitempty"` // +optional AdvertiseAddr string `json:"advertise-addr,omitempty" toml:"advertise-addr,omitempty"` - // Optional: Defaults to /data/proxy + // Optional: Defaults to /data0/proxy // +optional DataDir string `json:"data-dir,omitempty" toml:"data-dir,omitempty"` - // Optional: Defaults to /data/proxy.toml + // Optional: Defaults to /data0/proxy.toml // +optional Config string `json:"config,omitempty" toml:"config,omitempty"` - // Optional: Defaults to /data/logs/proxy.log + // Optional: Defaults to /data0/logs/proxy.log // +optional LogFile string `json:"log-file,omitempty" toml:"log-file,omitempty"` } diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index a3d1bfe8fa..5d9e4ce89c 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -42,6 +42,8 @@ const ( TiDBMemberType MemberType = "tidb" // TiKVMemberType is tikv container type TiKVMemberType MemberType = "tikv" + // TiFlashMemberType is tiflash container type + TiFlashMemberType MemberType = "tiflash" // SlowLogTailerMemberType is tidb log tailer container type SlowLogTailerMemberType MemberType = "slowlog" // UnknownMemberType is unknown container type @@ -203,11 +205,12 @@ type TidbClusterSpec struct { // TidbClusterStatus represents the current status of a tidb cluster. type TidbClusterStatus struct { - ClusterID string `json:"clusterID,omitempty"` - PD PDStatus `json:"pd,omitempty"` - TiKV TiKVStatus `json:"tikv,omitempty"` - TiDB TiDBStatus `json:"tidb,omitempty"` - Pump PumpStatus `josn:"pump,omitempty"` + ClusterID string `json:"clusterID,omitempty"` + PD PDStatus `json:"pd,omitempty"` + TiKV TiKVStatus `json:"tikv,omitempty"` + TiDB TiDBStatus `json:"tidb,omitempty"` + Pump PumpStatus `josn:"pump,omitempty"` + TiFlash TiFlashStatus `json:"tiflash,omitempty"` } // +k8s:openapi-gen=true @@ -319,6 +322,16 @@ type TiFlashSpec struct { // Config is the Configuration of TiFlash // +optional Config *TiFlashConfig `json:"config,omitempty"` + + // LogTailer is the configurations of the log tailers for TiFlash + // +optional + LogTailer *LogTailerSpec `json:"logTailer,omitempty"` +} + +// +k8s:openapi-gen=true +// LogTailerSpec represents an optional log tailer sidecar container +type LogTailerSpec struct { + corev1.ResourceRequirements `json:",inline"` } // +k8s:openapi-gen=true @@ -655,6 +668,17 @@ type TiKVStatus struct { Image string `json:"image,omitempty"` } +// TiFlashStatus is TiFlash status +type TiFlashStatus struct { + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Stores map[string]TiKVStore `json:"stores,omitempty"` + TombstoneStores map[string]TiKVStore `json:"tombstoneStores,omitempty"` + FailureStores map[string]TiKVFailureStore `json:"failureStores,omitempty"` + Image string `json:"image,omitempty"` +} + // TiKVStores is either Up/Down/Offline/Tombstone type TiKVStore struct { // store id is also uint64, due to the same reason as pd id, we store id as string diff --git a/pkg/apis/pingcap/v1alpha1/validation/validation.go b/pkg/apis/pingcap/v1alpha1/validation/validation.go index 1196fb8271..5f5fd799a3 100644 --- a/pkg/apis/pingcap/v1alpha1/validation/validation.go +++ b/pkg/apis/pingcap/v1alpha1/validation/validation.go @@ -16,7 +16,9 @@ package validation import ( "encoding/json" "fmt" + "os" "reflect" + "strings" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/label" @@ -78,6 +80,10 @@ func validateTiFlashSpec(spec *v1alpha1.TiFlashSpec, fldPath *field.Path) field. allErrs := field.ErrorList{} allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...) allErrs = append(allErrs, validateTiFlashConfig(spec.Config, fldPath)...) + if len(spec.StorageClaims) < 1 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("spec.StorageClaims"), + spec.StorageClaims, "storageClaims should be configured at least one item.")) + } return allErrs } @@ -86,11 +92,58 @@ func validateTiFlashConfig(config *v1alpha1.TiFlashConfig, path *field.Path) fie if config == nil { return allErrs } - if config.CommonConfig.Flash.OverlapThreshold != nil { - if *config.CommonConfig.Flash.OverlapThreshold < 0 || *config.CommonConfig.Flash.OverlapThreshold > 1 { - allErrs = append(allErrs, field.Invalid(path.Child("config.config.flash.overlap_threshold"), - config.CommonConfig.Flash.OverlapThreshold, - "overlap_threshold must be in the range of [0,1].")) + + if config.CommonConfig != nil { + if config.CommonConfig.Flash != nil { + if config.CommonConfig.Flash.OverlapThreshold != nil { + if *config.CommonConfig.Flash.OverlapThreshold < 0 || *config.CommonConfig.Flash.OverlapThreshold > 1 { + allErrs = append(allErrs, field.Invalid(path.Child("config.config.flash.overlap_threshold"), + config.CommonConfig.Flash.OverlapThreshold, + "overlap_threshold must be in the range of [0,1].")) + } + } + if config.CommonConfig.Flash.FlashCluster != nil { + if config.CommonConfig.Flash.FlashCluster.ClusterLog != "" { + splitPath := strings.Split(config.CommonConfig.Flash.FlashCluster.ClusterLog, string(os.PathSeparator)) + // The log path should be at least /dir/base.log + if len(splitPath) < 3 { + allErrs = append(allErrs, field.Invalid(path.Child("config.config.flash.flash_cluster.log"), + config.CommonConfig.Flash.FlashCluster.ClusterLog, + "log path should include at least one level dir.")) + } + } + } + if config.CommonConfig.Flash.FlashProxy != nil { + if config.CommonConfig.Flash.FlashProxy.LogFile != "" { + splitPath := strings.Split(config.CommonConfig.Flash.FlashProxy.LogFile, string(os.PathSeparator)) + // The log path should be at least /dir/base.log + if len(splitPath) < 3 { + allErrs = append(allErrs, field.Invalid(path.Child("config.config.flash.flash_proxy.log-file"), + config.CommonConfig.Flash.FlashProxy.LogFile, + "log path should include at least one level dir.")) + } + } + } + } + if config.CommonConfig.FlashLogger != nil { + if config.CommonConfig.FlashLogger.ServerLog != "" { + splitPath := strings.Split(config.CommonConfig.FlashLogger.ServerLog, string(os.PathSeparator)) + // The log path should be at least /dir/base.log + if len(splitPath) < 3 { + allErrs = append(allErrs, field.Invalid(path.Child("config.config.logger.log"), + config.CommonConfig.FlashLogger.ServerLog, + "log path should include at least one level dir.")) + } + } + if config.CommonConfig.FlashLogger.ErrorLog != "" { + splitPath := strings.Split(config.CommonConfig.FlashLogger.ErrorLog, string(os.PathSeparator)) + // The log path should be at least /dir/base.log + if len(splitPath) < 3 { + allErrs = append(allErrs, field.Invalid(path.Child("config.config.logger.errorlog"), + config.CommonConfig.FlashLogger.ErrorLog, + "log path should include at least one level dir.")) + } + } } } return allErrs diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index d2be3759ea..4d30fc65f6 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -1308,6 +1308,23 @@ func (in *Log) DeepCopy() *Log { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogTailerSpec) DeepCopyInto(out *LogTailerSpec) { + *out = *in + in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogTailerSpec. +func (in *LogTailerSpec) DeepCopy() *LogTailerSpec { + if in == nil { + return nil + } + out := new(LogTailerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MasterKeyFileConfig) DeepCopyInto(out *MasterKeyFileConfig) { *out = *in @@ -2500,22 +2517,6 @@ func (in *ProxyProtocol) DeepCopy() *ProxyProtocol { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProxyServer) DeepCopyInto(out *ProxyServer) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyServer. -func (in *ProxyServer) DeepCopy() *ProxyServer { - if in == nil { - return nil - } - out := new(ProxyServer) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PumpSpec) DeepCopyInto(out *PumpSpec) { *out = *in @@ -3497,6 +3498,11 @@ func (in *TiFlashSpec) DeepCopyInto(out *TiFlashSpec) { *out = new(TiFlashConfig) (*in).DeepCopyInto(*out) } + if in.LogTailer != nil { + in, out := &in.LogTailer, &out.LogTailer + *out = new(LogTailerSpec) + (*in).DeepCopyInto(*out) + } return } @@ -3510,6 +3516,48 @@ func (in *TiFlashSpec) DeepCopy() *TiFlashSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TiFlashStatus) DeepCopyInto(out *TiFlashStatus) { + *out = *in + if in.StatefulSet != nil { + in, out := &in.StatefulSet, &out.StatefulSet + *out = new(appsv1.StatefulSetStatus) + (*in).DeepCopyInto(*out) + } + if in.Stores != nil { + in, out := &in.Stores, &out.Stores + *out = make(map[string]TiKVStore, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.TombstoneStores != nil { + in, out := &in.TombstoneStores, &out.TombstoneStores + *out = make(map[string]TiKVStore, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.FailureStores != nil { + in, out := &in.FailureStores, &out.FailureStores + *out = make(map[string]TiKVFailureStore, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TiFlashStatus. +func (in *TiFlashStatus) DeepCopy() *TiFlashStatus { + if in == nil { + return nil + } + out := new(TiFlashStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TiKVBlockCacheConfig) DeepCopyInto(out *TiKVBlockCacheConfig) { *out = *in @@ -5045,6 +5093,7 @@ func (in *TidbClusterStatus) DeepCopyInto(out *TidbClusterStatus) { in.TiKV.DeepCopyInto(&out.TiKV) in.TiDB.DeepCopyInto(&out.TiDB) in.Pump.DeepCopyInto(&out.Pump) + in.TiFlash.DeepCopyInto(&out.TiFlash) return } diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index 442555f958..8671f7b5a9 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -47,6 +47,7 @@ func NewDefaultTidbClusterControl( orphanPodsCleaner member.OrphanPodsCleaner, pvcCleaner member.PVCCleanerInterface, pumpMemberManager manager.Manager, + tiflashMemberManager manager.Manager, discoveryManager member.TidbDiscoveryManager, podRestarter member.PodRestarter, recorder record.EventRecorder) ControlInterface { @@ -60,6 +61,7 @@ func NewDefaultTidbClusterControl( orphanPodsCleaner, pvcCleaner, pumpMemberManager, + tiflashMemberManager, discoveryManager, podRestarter, recorder, @@ -76,6 +78,7 @@ type defaultTidbClusterControl struct { orphanPodsCleaner member.OrphanPodsCleaner pvcCleaner member.PVCCleanerInterface pumpMemberManager manager.Manager + tiflashMemberManager manager.Manager discoveryManager member.TidbDiscoveryManager podRestarter member.PodRestarter recorder record.EventRecorder @@ -180,6 +183,19 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster return err } + // works that should do to making the tiflash cluster current state match the desired state: + // - waiting for the tidb cluster available + // - create or update tiflash headless service + // - create the tiflash statefulset + // - sync tiflash cluster status from pd to TidbCluster object + // - set scheduler labels to tiflash stores + // - upgrade the tiflash cluster + // - scale out/in the tiflash cluster + // - failover the tiflash cluster + if err := tcc.tiflashMemberManager.Sync(tc); err != nil { + return err + } + // syncing the labels from Pod to PVC and PV, these labels include: // - label.StoreIDLabelKey // - label.MemberIDLabelKey diff --git a/pkg/controller/tidbcluster/tidb_cluster_control_test.go b/pkg/controller/tidbcluster/tidb_cluster_control_test.go index f12593cb92..41a73f3a51 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control_test.go @@ -316,6 +316,7 @@ func newFakeTidbClusterControl() ( orphanPodCleaner := mm.NewFakeOrphanPodsCleaner() pvcCleaner := mm.NewFakePVCCleaner() pumpMemberManager := mm.NewFakePumpMemberManager() + tiflashMemberManager := mm.NewFakeTiFlashMemberManager() discoveryManager := mm.NewFakeDiscoveryManger() podRestarter := mm.NewFakePodRestarter() control := NewDefaultTidbClusterControl( @@ -328,6 +329,7 @@ func newFakeTidbClusterControl() ( orphanPodCleaner, pvcCleaner, pumpMemberManager, + tiflashMemberManager, discoveryManager, podRestarter, recorder, diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index dba0a9e7bb..a7dd4d3af5 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -75,6 +75,7 @@ func NewController( pdFailoverPeriod time.Duration, tikvFailoverPeriod time.Duration, tidbFailoverPeriod time.Duration, + tiflashFailoverPeriod time.Duration, ) *Controller { eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{QPS: 1}) eventBroadcaster.StartLogging(klog.V(2).Infof) @@ -106,11 +107,14 @@ func NewController( typedControl := controller.NewTypedControl(controller.NewRealGenericControl(genericCli, recorder)) pdScaler := mm.NewPDScaler(pdControl, pvcInformer.Lister(), pvcControl) tikvScaler := mm.NewTiKVScaler(pdControl, pvcInformer.Lister(), pvcControl, podInformer.Lister()) + tiflashScaler := mm.NewTiFlashScaler(pdControl, pvcInformer.Lister(), pvcControl, podInformer.Lister()) pdFailover := mm.NewPDFailover(cli, pdControl, pdFailoverPeriod, podInformer.Lister(), podControl, pvcInformer.Lister(), pvcControl, pvInformer.Lister(), recorder) tikvFailover := mm.NewTiKVFailover(tikvFailoverPeriod) + tiflashFailover := mm.NewTiFlashFailover(tiflashFailoverPeriod) tidbFailover := mm.NewTiDBFailover(tidbFailoverPeriod) pdUpgrader := mm.NewPDUpgrader(pdControl, podControl, podInformer.Lister()) tikvUpgrader := mm.NewTiKVUpgrader(pdControl, podControl, podInformer.Lister()) + tiflashUpgrader := mm.NewTiFlashUpgrader(pdControl, podControl, podInformer.Lister()) tidbUpgrader := mm.NewTiDBUpgrader(tidbControl, podInformer.Lister()) podRestarter := mm.NewPodRestarter(kubeCli, podInformer.Lister()) @@ -201,6 +205,21 @@ func NewController( svcInformer.Lister(), podInformer.Lister(), ), + mm.NewTiFlashMemberManager( + pdControl, + setControl, + svcControl, + certControl, + typedControl, + setInformer.Lister(), + svcInformer.Lister(), + podInformer.Lister(), + nodeInformer.Lister(), + autoFailover, + tiflashFailover, + tiflashScaler, + tiflashUpgrader, + ), mm.NewTidbDiscoveryManager(typedControl), podRestarter, recorder, diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go index f666195dd9..a3639023be 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller_test.go @@ -281,6 +281,7 @@ func newFakeTidbClusterController() (*Controller, cache.Indexer, *FakeTidbCluste 5*time.Minute, 5*time.Minute, 5*time.Minute, + 5*time.Minute, ) tcc.tcListerSynced = alwaysReady tcc.setListerSynced = alwaysReady diff --git a/pkg/label/label.go b/pkg/label/label.go index 7d0b06787a..472dc70810 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -130,6 +130,8 @@ const ( TiDBLabelVal string = "tidb" // TiKVLabelVal is TiKV label value TiKVLabelVal string = "tikv" + // TiFlashLabelVal is TiKV label value + TiFlashLabelVal string = "tiflash" // PumpLabelVal is Pump label value PumpLabelVal string = "pump" // DiscoveryLabelVal is Discovery label value @@ -307,6 +309,12 @@ func (l Label) TiKV() Label { return l } +// TiFlash assigns tiflash to component key in label +func (l Label) TiFlash() Label { + l.Component(TiFlashLabelVal) + return l +} + // IsTiKV returns whether label is a TiKV func (l Label) IsTiKV() bool { return l[ComponentLabelKey] == TiKVLabelVal diff --git a/pkg/manager/member/tiflash_failover.go b/pkg/manager/member/tiflash_failover.go new file mode 100644 index 0000000000..08ffa919dd --- /dev/null +++ b/pkg/manager/member/tiflash_failover.go @@ -0,0 +1,53 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "time" + + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" +) + +type tiflashFailover struct { + tiflashFailoverPeriod time.Duration +} + +// NewTiFlashFailover returns a tiflash Failover +func NewTiFlashFailover(tiflashFailoverPeriod time.Duration) Failover { + return &tiflashFailover{tiflashFailoverPeriod} +} + +// TODO: Finish the failover logic +func (tff *tiflashFailover) Failover(tc *v1alpha1.TidbCluster) error { + return nil +} + +func (tff *tiflashFailover) Recover(_ *v1alpha1.TidbCluster) { + // Do nothing now +} + +type fakeTiFlashFailover struct{} + +// NewFakeTiFlashFailover returns a fake Failover +func NewFakeTiFlashFailover() Failover { + return &fakeTiFlashFailover{} +} + +func (ftff *fakeTiFlashFailover) Failover(_ *v1alpha1.TidbCluster) error { + return nil +} + +func (ftff *fakeTiFlashFailover) Recover(_ *v1alpha1.TidbCluster) { + return +} diff --git a/pkg/manager/member/tiflash_member_manager.go b/pkg/manager/member/tiflash_member_manager.go new file mode 100644 index 0000000000..f7a57e12ec --- /dev/null +++ b/pkg/manager/member/tiflash_member_manager.go @@ -0,0 +1,858 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "fmt" + "reflect" + "regexp" + "strings" + + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/manager" + "github.com/pingcap/tidb-operator/pkg/pdapi" + "github.com/pingcap/tidb-operator/pkg/util" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" + v1 "k8s.io/client-go/listers/apps/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/klog" +) + +const ( + // tiflashClusterCertPath is where the cert for inter-cluster communication stored (if any) + tiflashClusterCertPath = "/var/lib/tiflash-tls" + + //find a better way to manage store only managed by tiflash in Operator + tiflashStoreLimitPattern = `%s-tiflash-\d+\.%s-tiflash-peer\.%s\.svc\:\d+` +) + +// tiflashMemberManager implements manager.Manager. +type tiflashMemberManager struct { + setControl controller.StatefulSetControlInterface + svcControl controller.ServiceControlInterface + pdControl pdapi.PDControlInterface + certControl controller.CertControlInterface + typedControl controller.TypedControlInterface + setLister v1.StatefulSetLister + svcLister corelisters.ServiceLister + podLister corelisters.PodLister + nodeLister corelisters.NodeLister + autoFailover bool + tiflashFailover Failover + tiflashScaler Scaler + tiflashUpgrader Upgrader + tiflashStatefulSetIsUpgradingFn func(corelisters.PodLister, pdapi.PDControlInterface, *apps.StatefulSet, *v1alpha1.TidbCluster) (bool, error) +} + +// NewTiFlashMemberManager returns a *tiflashMemberManager +func NewTiFlashMemberManager( + pdControl pdapi.PDControlInterface, + setControl controller.StatefulSetControlInterface, + svcControl controller.ServiceControlInterface, + certControl controller.CertControlInterface, + typedControl controller.TypedControlInterface, + setLister v1.StatefulSetLister, + svcLister corelisters.ServiceLister, + podLister corelisters.PodLister, + nodeLister corelisters.NodeLister, + autoFailover bool, + tiflashFailover Failover, + tiflashScaler Scaler, + tiflashUpgrader Upgrader) manager.Manager { + kvmm := tiflashMemberManager{ + pdControl: pdControl, + podLister: podLister, + nodeLister: nodeLister, + setControl: setControl, + svcControl: svcControl, + certControl: certControl, + typedControl: typedControl, + setLister: setLister, + svcLister: svcLister, + autoFailover: autoFailover, + tiflashFailover: tiflashFailover, + tiflashScaler: tiflashScaler, + tiflashUpgrader: tiflashUpgrader, + } + kvmm.tiflashStatefulSetIsUpgradingFn = tiflashStatefulSetIsUpgrading + return &kvmm +} + +// Sync fulfills the manager.Manager interface +func (tfmm *tiflashMemberManager) Sync(tc *v1alpha1.TidbCluster) error { + if tc.Spec.TiFlash == nil { + return nil + } + + ns := tc.GetNamespace() + tcName := tc.GetName() + + if !tc.PDIsAvailable() { + return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) + } + + // Sync TiFlash Headless Service + if err := tfmm.syncHeadlessService(tc); err != nil { + return err + } + + return tfmm.syncStatefulSet(tc) +} + +func (tfmm *tiflashMemberManager) syncHeadlessService(tc *v1alpha1.TidbCluster) error { + if tc.Spec.Paused { + klog.V(4).Infof("tiflash cluster %s/%s is paused, skip syncing for tiflash service", tc.GetNamespace(), tc.GetName()) + return nil + } + + ns := tc.GetNamespace() + tcName := tc.GetName() + + newSvc := getNewHeadlessService(tc) + oldSvcTmp, err := tfmm.svcLister.Services(ns).Get(controller.TiFlashPeerMemberName(tcName)) + if errors.IsNotFound(err) { + err = controller.SetServiceLastAppliedConfigAnnotation(newSvc) + if err != nil { + return err + } + return tfmm.svcControl.CreateService(tc, newSvc) + } + if err != nil { + return err + } + + oldSvc := oldSvcTmp.DeepCopy() + + equal, err := controller.ServiceEqual(newSvc, oldSvc) + if err != nil { + return err + } + if !equal { + svc := *oldSvc + svc.Spec = newSvc.Spec + err = controller.SetServiceLastAppliedConfigAnnotation(newSvc) + if err != nil { + return err + } + _, err = tfmm.svcControl.UpdateService(tc, &svc) + return err + } + + return nil +} + +func (tfmm *tiflashMemberManager) syncStatefulSet(tc *v1alpha1.TidbCluster) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + + oldSetTmp, err := tfmm.setLister.StatefulSets(ns).Get(controller.TiFlashMemberName(tcName)) + if err != nil && !errors.IsNotFound(err) { + return err + } + setNotExist := errors.IsNotFound(err) + + oldSet := oldSetTmp.DeepCopy() + + if err := tfmm.syncTidbClusterStatus(tc, oldSet); err != nil { + return err + } + + if tc.Spec.Paused { + klog.V(4).Infof("tiflash cluster %s/%s is paused, skip syncing for tiflash statefulset", tc.GetNamespace(), tc.GetName()) + return nil + } + + cm, err := tfmm.syncConfigMap(tc, oldSet) + if err != nil { + return err + } + + newSet, err := getNewStatefulSet(tc, cm) + if err != nil { + return err + } + if setNotExist { + err = SetStatefulSetLastAppliedConfigAnnotation(newSet) + if err != nil { + return err + } + err = tfmm.setControl.CreateStatefulSet(tc, newSet) + if err != nil { + return err + } + tc.Status.TiFlash.StatefulSet = &apps.StatefulSetStatus{} + return nil + } + + if _, err := tfmm.setStoreLabelsForTiFlash(tc); err != nil { + return err + } + + if !templateEqual(newSet, oldSet) || tc.Status.TiFlash.Phase == v1alpha1.UpgradePhase { + if err := tfmm.tiflashUpgrader.Upgrade(tc, oldSet, newSet); err != nil { + return err + } + } + + if err := tfmm.tiflashScaler.Scale(tc, oldSet, newSet); err != nil { + return err + } + + if tfmm.autoFailover && tc.Spec.TiFlash.MaxFailoverCount != nil { + if tc.TiFlashAllPodsStarted() && !tc.TiFlashAllStoresReady() { + if err := tfmm.tiflashFailover.Failover(tc); err != nil { + return err + } + } + } + + return updateStatefulSet(tfmm.setControl, tc, newSet, oldSet) +} + +func (tfmm *tiflashMemberManager) syncConfigMap(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) (*corev1.ConfigMap, error) { + newCm, err := getTiFlashConfigMap(tc) + if err != nil { + return nil, err + } + if set != nil && tc.BaseTiFlashSpec().ConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyInPlace { + inUseName := FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool { + return strings.HasPrefix(name, controller.TiFlashMemberName(tc.Name)) + }) + if inUseName != "" { + newCm.Name = inUseName + } + } + + return tfmm.typedControl.CreateOrUpdateConfigMap(tc, newCm) +} + +func getNewHeadlessService(tc *v1alpha1.TidbCluster) *corev1.Service { + ns := tc.Namespace + tcName := tc.Name + instanceName := tc.GetInstanceName() + svcName := controller.TiFlashPeerMemberName(tcName) + svcLabel := label.New().Instance(instanceName).TiFlash().Labels() + + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + Namespace: ns, + Labels: svcLabel, + OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "tiflash", + Port: 3930, + TargetPort: intstr.FromInt(int(3930)), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "proxy", + Port: 20170, + TargetPort: intstr.FromInt(int(20170)), + Protocol: corev1.ProtocolTCP, + }, + }, + Selector: svcLabel, + PublishNotReadyAddresses: true, + }, + } + return &svc +} + +func getNewStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.StatefulSet, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + baseTiFlashSpec := tc.BaseTiFlashSpec() + spec := tc.Spec.TiFlash + + tiflashConfigMap := controller.MemberConfigMapName(tc, v1alpha1.TiFlashMemberType) + if cm != nil { + tiflashConfigMap = cm.Name + } + + // This should not happen as we have validaton for this field + if len(spec.StorageClaims) < 1 { + return nil, fmt.Errorf("storageClaims should be configured at least one item for tiflash, tidbcluster %s/%s", tc.Namespace, tc.Name) + } + pvcs, err := flashVolumeClaimTemplate(tc.Spec.TiFlash.StorageClaims) + if err != nil { + return nil, fmt.Errorf("cannot parse storage request for tiflash.StorageClaims, tidbcluster %s/%s, error: %v", tc.Namespace, tc.Name, err) + } + annMount, annVolume := annotationsMountVolume() + volMounts := []corev1.VolumeMount{ + annMount, + } + for k := range spec.StorageClaims { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: fmt.Sprintf("data%d", k), MountPath: fmt.Sprintf("/data%d", k)}) + } + + // TiFlash does not support TLS yet + // if tc.IsTLSClusterEnabled() { + // volMounts = append(volMounts, corev1.VolumeMount{ + // Name: "tiflash-tls", ReadOnly: true, MountPath: "/var/lib/tiflash-tls", + // }) + // } + + vols := []corev1.Volume{ + annVolume, + {Name: "config", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tiflashConfigMap, + }, + }}, + }, + } + + // if tc.IsTLSClusterEnabled() { + // vols = append(vols, corev1.Volume{ + // Name: "tiflash-tls", VolumeSource: corev1.VolumeSource{ + // Secret: &corev1.SecretVolumeSource{ + // SecretName: util.ClusterTLSSecretName(tc.Name, label.TiFlashLabelVal), + // }, + // }, + // }) + // } + + sysctls := "sysctl -w" + var initContainers []corev1.Container + if baseTiFlashSpec.Annotations() != nil { + init, ok := baseTiFlashSpec.Annotations()[label.AnnSysctlInit] + if ok && (init == label.AnnSysctlInitVal) { + if baseTiFlashSpec.PodSecurityContext() != nil && len(baseTiFlashSpec.PodSecurityContext().Sysctls) > 0 { + for _, sysctl := range baseTiFlashSpec.PodSecurityContext().Sysctls { + sysctls = sysctls + fmt.Sprintf(" %s=%s", sysctl.Name, sysctl.Value) + } + privileged := true + initContainers = append(initContainers, corev1.Container{ + Name: "init", + Image: tc.HelperImage(), + Command: []string{ + "sh", + "-c", + sysctls, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }) + } + } + } + // Init container is only used for the case where allowed-unsafe-sysctls + // cannot be enabled for kubelet, so clean the sysctl in statefulset + // SecurityContext if init container is enabled + podSecurityContext := baseTiFlashSpec.PodSecurityContext().DeepCopy() + if len(initContainers) > 0 { + podSecurityContext.Sysctls = []corev1.Sysctl{} + } + + // Append init container for config files initialization + initVolMounts := []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + {Name: "config", ReadOnly: true, MountPath: "/etc/tiflash"}, + } + initEnv := []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + } + initContainers = append(initContainers, corev1.Container{ + Name: "init", + Image: tc.HelperImage(), + Command: []string{ + "sh", + "-c", + "set -ex;ordinal=`echo ${POD_NAME} | awk -F- '{print $NF}'`;sed s/POD_NUM/${ordinal}/g /etc/tiflash/config_templ.toml > /data0/config.toml;sed s/POD_NUM/${ordinal}/g /etc/tiflash/proxy_templ.toml > /data0/proxy.toml", + }, + Env: initEnv, + VolumeMounts: initVolMounts, + }) + + tiflashLabel := labelTiFlash(tc) + setName := controller.TiFlashMemberName(tcName) + podAnnotations := CombineAnnotations(controller.AnnProm(8234), baseTiFlashSpec.Annotations()) + stsAnnotations := getStsAnnotations(tc, label.TiFlashLabelVal) + capacity := controller.TiKVCapacity(tc.Spec.TiFlash.Limits) + headlessSvcName := controller.TiFlashPeerMemberName(tcName) + + env := []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "CLUSTER_NAME", + Value: tcName, + }, + { + Name: "HEADLESS_SERVICE_NAME", + Value: headlessSvcName, + }, + { + Name: "CAPACITY", + Value: capacity, + }, + { + Name: "TZ", + Value: tc.Spec.Timezone, + }, + } + tiflashContainer := corev1.Container{ + Name: v1alpha1.TiFlashMemberType.String(), + Image: tc.TiFlashImage(), + ImagePullPolicy: baseTiFlashSpec.ImagePullPolicy(), + Command: []string{"/bin/sh", "-c", "/tiflash/tiflash server --config-file /data0/config.toml"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: tc.TiFlashContainerPrivilege(), + }, + Ports: []corev1.ContainerPort{ + { + Name: "tiflash", + ContainerPort: int32(3930), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "proxy", + ContainerPort: int32(20170), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "tcp", + ContainerPort: int32(9000), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "http", + ContainerPort: int32(8123), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "internal", + ContainerPort: int32(9009), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "metrics", + ContainerPort: int32(8234), + Protocol: corev1.ProtocolTCP, + }, + }, + VolumeMounts: volMounts, + Resources: controller.ContainerResource(tc.Spec.TiFlash.ResourceRequirements), + } + podSpec := baseTiFlashSpec.BuildPodSpec() + if baseTiFlashSpec.HostNetwork() { + podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + env = append(env, corev1.EnvVar{ + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }) + } + tiflashContainer.Env = util.AppendEnv(env, baseTiFlashSpec.Env()) + podSpec.Volumes = vols + podSpec.SecurityContext = podSecurityContext + podSpec.InitContainers = initContainers + podSpec.Containers = []corev1.Container{tiflashContainer} + podSpec.Containers = append(podSpec.Containers, buildTiFlashSidecarContainers(tc)...) + podSpec.ServiceAccountName = tc.Spec.TiFlash.ServiceAccount + + tiflashset := &apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: setName, + Namespace: ns, + Labels: tiflashLabel.Labels(), + Annotations: stsAnnotations, + OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, + }, + Spec: apps.StatefulSetSpec{ + Replicas: controller.Int32Ptr(tc.TiFlashStsDesiredReplicas()), + Selector: tiflashLabel.LabelSelector(), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: tiflashLabel.Labels(), + Annotations: podAnnotations, + }, + Spec: podSpec, + }, + VolumeClaimTemplates: pvcs, + ServiceName: headlessSvcName, + PodManagementPolicy: apps.ParallelPodManagement, + UpdateStrategy: apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &apps.RollingUpdateStatefulSetStrategy{ + Partition: controller.Int32Ptr(tc.TiFlashStsDesiredReplicas()), + }, + }, + }, + } + return tiflashset, nil +} + +func flashVolumeClaimTemplate(storageClaims []v1alpha1.StorageClaim) ([]corev1.PersistentVolumeClaim, error) { + var pvcs []corev1.PersistentVolumeClaim + for k := range storageClaims { + storageRequest, err := controller.ParseStorageRequest(storageClaims[k].Resources.Requests) + if err != nil { + return nil, err + } + pvcs = append(pvcs, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("data%d", k)}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + StorageClassName: storageClaims[k].StorageClassName, + Resources: storageRequest, + }, + }) + } + return pvcs, nil +} + +func getTiFlashConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { + config := tc.Spec.TiFlash.Config.DeepCopy() + if config == nil { + config = &v1alpha1.TiFlashConfig{} + } + setTiFlashConfigDefault(config, tc.Name, tc.Namespace) + + // override CA if tls enabled + // if tc.IsTLSClusterEnabled() { + // if config.Security == nil { + // config.Security = &v1alpha1.TiFlashSecurityConfig{} + // } + // config.Security.CAPath = path.Join(tiflashClusterCertPath, tlsSecretRootCAKey) + // config.Security.CertPath = path.Join(tiflashClusterCertPath, corev1.TLSCertKey) + // config.Security.KeyPath = path.Join(tiflashClusterCertPath, corev1.TLSPrivateKeyKey) + // } + + configText, err := MarshalTOML(config.CommonConfig) + if err != nil { + return nil, err + } + proxyText, err := MarshalTOML(config.ProxyConfig) + if err != nil { + return nil, err + } + + instanceName := tc.GetInstanceName() + tiflashLabel := label.New().Instance(instanceName).TiFlash().Labels() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.TiFlashMemberName(tc.Name), + Namespace: tc.Namespace, + Labels: tiflashLabel, + OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(tc)}, + }, + Data: map[string]string{ + "config_templ.toml": string(configText), + "proxy_templ.toml": string(proxyText), + }, + } + + if tc.BaseTiFlashSpec().ConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { + if err := AddConfigMapDigestSuffix(cm); err != nil { + return nil, err + } + } + + return cm, nil +} + +func labelTiFlash(tc *v1alpha1.TidbCluster) label.Label { + instanceName := tc.GetInstanceName() + return label.New().Instance(instanceName).TiFlash() +} + +func (tfmm *tiflashMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) error { + if set == nil { + // skip if not created yet + return nil + } + tc.Status.TiFlash.StatefulSet = &set.Status + upgrading, err := tfmm.tiflashStatefulSetIsUpgradingFn(tfmm.podLister, tfmm.pdControl, set, tc) + if err != nil { + return err + } + if upgrading && tc.Status.PD.Phase != v1alpha1.UpgradePhase { + tc.Status.TiFlash.Phase = v1alpha1.UpgradePhase + } else { + tc.Status.TiFlash.Phase = v1alpha1.NormalPhase + } + + previousStores := tc.Status.TiFlash.Stores + stores := map[string]v1alpha1.TiKVStore{} + tombstoneStores := map[string]v1alpha1.TiKVStore{} + + pdCli := controller.GetPDClient(tfmm.pdControl, tc) + // This only returns Up/Down/Offline stores + storesInfo, err := pdCli.GetStores() + if err != nil { + tc.Status.TiFlash.Synced = false + return err + } + + pattern, err := regexp.Compile(fmt.Sprintf(tiflashStoreLimitPattern, tc.Name, tc.Name, tc.Namespace)) + if err != nil { + return err + } + for _, store := range storesInfo.Stores { + // In theory, the external tiflash can join the cluster, and the operator would only manage the internal tiflash. + // So we check the store owner to make sure it. + if store.Store != nil && !pattern.Match([]byte(store.Store.Address)) { + continue + } + status := tfmm.getTiFlashStore(store) + if status == nil { + continue + } + // avoid LastHeartbeatTime be overwrite by zero time when pd lost LastHeartbeatTime + if status.LastHeartbeatTime.IsZero() { + if oldStatus, ok := previousStores[status.ID]; ok { + klog.V(4).Infof("the pod:%s's store LastHeartbeatTime is zero,so will keep in %v", status.PodName, oldStatus.LastHeartbeatTime) + status.LastHeartbeatTime = oldStatus.LastHeartbeatTime + } + } + + oldStore, exist := previousStores[status.ID] + + status.LastTransitionTime = metav1.Now() + if exist && status.State == oldStore.State { + status.LastTransitionTime = oldStore.LastTransitionTime + } + + stores[status.ID] = *status + } + + //this returns all tombstone stores + tombstoneStoresInfo, err := pdCli.GetTombStoneStores() + if err != nil { + tc.Status.TiFlash.Synced = false + return err + } + for _, store := range tombstoneStoresInfo.Stores { + status := tfmm.getTiFlashStore(store) + if status == nil { + continue + } + tombstoneStores[status.ID] = *status + } + + tc.Status.TiFlash.Synced = true + tc.Status.TiFlash.Stores = stores + tc.Status.TiFlash.TombstoneStores = tombstoneStores + tc.Status.TiFlash.Image = "" + c := filterContainer(set, "tiflash") + if c != nil { + tc.Status.TiFlash.Image = c.Image + } + return nil +} + +func (tfmm *tiflashMemberManager) getTiFlashStore(store *pdapi.StoreInfo) *v1alpha1.TiKVStore { + if store.Store == nil || store.Status == nil { + return nil + } + storeID := fmt.Sprintf("%d", store.Store.GetId()) + ip := strings.Split(store.Store.GetAddress(), ":")[0] + podName := strings.Split(ip, ".")[0] + + return &v1alpha1.TiKVStore{ + ID: storeID, + PodName: podName, + IP: ip, + LeaderCount: int32(store.Status.LeaderCount), + State: store.Store.StateName, + LastHeartbeatTime: metav1.Time{Time: store.Status.LastHeartbeatTS}, + } +} + +func (tfmm *tiflashMemberManager) setStoreLabelsForTiFlash(tc *v1alpha1.TidbCluster) (int, error) { + ns := tc.GetNamespace() + // for unit test + setCount := 0 + + pdCli := controller.GetPDClient(tfmm.pdControl, tc) + storesInfo, err := pdCli.GetStores() + if err != nil { + return setCount, err + } + + config, err := pdCli.GetConfig() + if err != nil { + return setCount, err + } + + locationLabels := []string(config.Replication.LocationLabels) + if locationLabels == nil { + return setCount, nil + } + + pattern, err := regexp.Compile(fmt.Sprintf(tiflashStoreLimitPattern, tc.Name, tc.Name, tc.Namespace)) + if err != nil { + return -1, err + } + for _, store := range storesInfo.Stores { + // In theory, the external tiflash can join the cluster, and the operator would only manage the internal tiflash. + // So we check the store owner to make sure it. + if store.Store != nil && !pattern.Match([]byte(store.Store.Address)) { + continue + } + status := tfmm.getTiFlashStore(store) + if status == nil { + continue + } + podName := status.PodName + + pod, err := tfmm.podLister.Pods(ns).Get(podName) + if err != nil { + return setCount, err + } + + nodeName := pod.Spec.NodeName + ls, err := tfmm.getNodeLabels(nodeName, locationLabels) + if err != nil || len(ls) == 0 { + klog.Warningf("node: [%s] has no node labels, skipping set store labels for Pod: [%s/%s]", nodeName, ns, podName) + continue + } + + if !tfmm.storeLabelsEqualNodeLabels(store.Store.Labels, ls) { + set, err := pdCli.SetStoreLabels(store.Store.Id, ls) + if err != nil { + klog.Warningf("failed to set pod: [%s/%s]'s store labels: %v", ns, podName, ls) + continue + } + if set { + setCount++ + klog.Infof("pod: [%s/%s] set labels: %v successfully", ns, podName, ls) + } + } + } + + return setCount, nil +} + +func (tfmm *tiflashMemberManager) getNodeLabels(nodeName string, storeLabels []string) (map[string]string, error) { + node, err := tfmm.nodeLister.Get(nodeName) + if err != nil { + return nil, err + } + labels := map[string]string{} + ls := node.GetLabels() + for _, storeLabel := range storeLabels { + if value, found := ls[storeLabel]; found { + labels[storeLabel] = value + continue + } + + // TODO after pd supports storeLabel containing slash character, these codes should be deleted + if storeLabel == "host" { + if host, found := ls[corev1.LabelHostname]; found { + labels[storeLabel] = host + } + } + + } + return labels, nil +} + +// storeLabelsEqualNodeLabels compares store labels with node labels +// for historic reasons, PD stores TiFlash labels as []*StoreLabel which is a key-value pair slice +func (tfmm *tiflashMemberManager) storeLabelsEqualNodeLabels(storeLabels []*metapb.StoreLabel, nodeLabels map[string]string) bool { + ls := map[string]string{} + for _, label := range storeLabels { + key := label.GetKey() + if _, ok := nodeLabels[key]; ok { + val := label.GetValue() + ls[key] = val + } + } + return reflect.DeepEqual(ls, nodeLabels) +} + +func tiflashStatefulSetIsUpgrading(podLister corelisters.PodLister, pdControl pdapi.PDControlInterface, set *apps.StatefulSet, tc *v1alpha1.TidbCluster) (bool, error) { + if statefulSetIsUpgrading(set) { + return true, nil + } + instanceName := tc.GetInstanceName() + selector, err := label.New().Instance(instanceName).TiFlash().Selector() + if err != nil { + return false, err + } + tiflashPods, err := podLister.Pods(tc.GetNamespace()).List(selector) + if err != nil { + return false, err + } + for _, pod := range tiflashPods { + revisionHash, exist := pod.Labels[apps.ControllerRevisionHashLabelKey] + if !exist { + return false, nil + } + if revisionHash != tc.Status.TiFlash.StatefulSet.UpdateRevision { + return true, nil + } + } + + return false, nil +} + +type FakeTiFlashMemberManager struct { + err error +} + +func NewFakeTiFlashMemberManager() *FakeTiFlashMemberManager { + return &FakeTiFlashMemberManager{} +} + +func (ftmm *FakeTiFlashMemberManager) SetSyncError(err error) { + ftmm.err = err +} + +func (ftmm *FakeTiFlashMemberManager) Sync(tc *v1alpha1.TidbCluster) error { + if ftmm.err != nil { + return ftmm.err + } + if len(tc.Status.TiFlash.Stores) != 0 { + // simulate status update + tc.Status.ClusterID = string(uuid.NewUUID()) + } + return nil +} diff --git a/pkg/manager/member/tiflash_scaler.go b/pkg/manager/member/tiflash_scaler.go new file mode 100644 index 0000000000..0b3dff0024 --- /dev/null +++ b/pkg/manager/member/tiflash_scaler.go @@ -0,0 +1,93 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/pdapi" + apps "k8s.io/api/apps/v1" + corelisters "k8s.io/client-go/listers/core/v1" +) + +type tiflashScaler struct { + generalScaler + podLister corelisters.PodLister +} + +// NewTiFlashScaler returns a tiflash Scaler +func NewTiFlashScaler(pdControl pdapi.PDControlInterface, + pvcLister corelisters.PersistentVolumeClaimLister, + pvcControl controller.PVCControlInterface, + podLister corelisters.PodLister) Scaler { + return &tiflashScaler{generalScaler{pdControl, pvcLister, pvcControl}, podLister} +} + +// TODO: Finish the scaling logic +func (tfs *tiflashScaler) Scale(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + scaling, _, _, _ := scaleOne(oldSet, newSet) + if scaling > 0 { + return tfs.ScaleOut(tc, oldSet, newSet) + } else if scaling < 0 { + return tfs.ScaleIn(tc, oldSet, newSet) + } + // we only sync auto scaler annotations when we are finishing syncing scaling + return tfs.SyncAutoScalerAnn(tc, oldSet) +} + +func (tfs *tiflashScaler) ScaleOut(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + + return nil +} + +func (tfs *tiflashScaler) ScaleIn(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + + return nil +} + +// SyncAutoScalerAnn reclaims the auto-scaling-out slots if the target pods no longer exist +func (tfs *tiflashScaler) SyncAutoScalerAnn(tc *v1alpha1.TidbCluster, actual *apps.StatefulSet) error { + + return nil +} + +type fakeTiFlashScaler struct{} + +// NewFakeTiFlashScaler returns a fake tiflash Scaler +func NewFakeTiFlashScaler() Scaler { + return &fakeTiFlashScaler{} +} + +func (fsd *fakeTiFlashScaler) Scale(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + if *newSet.Spec.Replicas > *oldSet.Spec.Replicas { + return fsd.ScaleOut(tc, oldSet, newSet) + } else if *newSet.Spec.Replicas < *oldSet.Spec.Replicas { + return fsd.ScaleIn(tc, oldSet, newSet) + } + return nil +} + +func (fsd *fakeTiFlashScaler) ScaleOut(_ *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + setReplicasAndDeleteSlots(newSet, *oldSet.Spec.Replicas+1, nil) + return nil +} + +func (fsd *fakeTiFlashScaler) ScaleIn(_ *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + setReplicasAndDeleteSlots(newSet, *oldSet.Spec.Replicas-1, nil) + return nil +} + +func (fsd *fakeTiFlashScaler) SyncAutoScalerAnn(tc *v1alpha1.TidbCluster, actual *apps.StatefulSet) error { + return nil +} diff --git a/pkg/manager/member/tiflash_upgrader.go b/pkg/manager/member/tiflash_upgrader.go new file mode 100644 index 0000000000..1d3ff7c824 --- /dev/null +++ b/pkg/manager/member/tiflash_upgrader.go @@ -0,0 +1,57 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/pdapi" + apps "k8s.io/api/apps/v1" + corelisters "k8s.io/client-go/listers/core/v1" +) + +type tiflashUpgrader struct { + pdControl pdapi.PDControlInterface + podControl controller.PodControlInterface + podLister corelisters.PodLister +} + +// NewTiFlashUpgrader returns a tiflash Upgrader +func NewTiFlashUpgrader(pdControl pdapi.PDControlInterface, + podControl controller.PodControlInterface, + podLister corelisters.PodLister) Upgrader { + return &tiflashUpgrader{ + pdControl: pdControl, + podControl: podControl, + podLister: podLister, + } +} + +// TODO: Finish the upgrade logic +func (tku *tiflashUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet, newSet *apps.StatefulSet) error { + + return nil +} + +type fakeTiFlashUpgrader struct{} + +// NewFakeTiFlashUpgrader returns a fake tiflash upgrader +func NewFakeTiFlashUpgrader() Upgrader { + return &fakeTiFlashUpgrader{} +} + +func (tku *fakeTiFlashUpgrader) Upgrade(tc *v1alpha1.TidbCluster, _ *apps.StatefulSet, _ *apps.StatefulSet) error { + tc.Status.TiFlash.Phase = v1alpha1.UpgradePhase + return nil +} diff --git a/pkg/manager/member/tiflash_util.go b/pkg/manager/member/tiflash_util.go new file mode 100644 index 0000000000..7e32223d92 --- /dev/null +++ b/pkg/manager/member/tiflash_util.go @@ -0,0 +1,419 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "fmt" + "os" + "strings" + + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" + corev1 "k8s.io/api/core/v1" +) + +const ( + defaultClusterLog = "/data0/logs/flash_cluster_manager.log" + defaultProxyLog = "/data0/logs/proxy.log" + defaultErrorLog = "/data0/logs/error.log" + defaultServerLog = "/data0/logs/server.log" +) + +func buildTiFlashSidecarContainers(tc *v1alpha1.TidbCluster) []corev1.Container { + spec := tc.Spec.TiFlash + config := spec.Config.DeepCopy() + image := tc.HelperImage() + pullPolicy := tc.HelperImagePullPolicy() + var containers []corev1.Container + var resource corev1.ResourceRequirements + if spec.LogTailer != nil { + resource = controller.ContainerResource(spec.LogTailer.ResourceRequirements) + } + if config == nil { + config = &v1alpha1.TiFlashConfig{} + } + setTiFlashLogConfigDefault(config) + containers = append(containers, buildSidecarContainer("serverlog", config.CommonConfig.FlashLogger.ServerLog, image, pullPolicy, resource)) + containers = append(containers, buildSidecarContainer("errorlog", config.CommonConfig.FlashLogger.ErrorLog, image, pullPolicy, resource)) + containers = append(containers, buildSidecarContainer("proxylog", config.CommonConfig.Flash.FlashProxy.LogFile, image, pullPolicy, resource)) + containers = append(containers, buildSidecarContainer("clusterlog", config.CommonConfig.Flash.FlashCluster.ClusterLog, image, pullPolicy, resource)) + return containers +} + +func buildSidecarContainer(name, path, image string, + pullPolicy corev1.PullPolicy, + resource corev1.ResourceRequirements) corev1.Container { + splitPath := strings.Split(path, string(os.PathSeparator)) + // The log path should be at least /dir/base.log + if len(splitPath) >= 3 { + serverLogVolumeName := splitPath[1] + serverLogMountDir := "/" + serverLogVolumeName + return corev1.Container{ + Name: name, + Image: image, + ImagePullPolicy: pullPolicy, + Resources: resource, + VolumeMounts: []corev1.VolumeMount{ + {Name: serverLogVolumeName, MountPath: serverLogMountDir}, + }, + Command: []string{ + "sh", + "-c", + fmt.Sprintf("touch %s; tail -n0 -F %s;", path, path), + }, + } + } + return corev1.Container{ + Name: name, + Image: image, + ImagePullPolicy: pullPolicy, + Resources: resource, + Command: []string{ + "sh", + "-c", + fmt.Sprintf("touch %s; tail -n0 -F %s;", path, path), + }, + } +} + +func setTiFlashLogConfigDefault(config *v1alpha1.TiFlashConfig) { + if config.CommonConfig == nil { + config.CommonConfig = &v1alpha1.CommonConfig{} + } + if config.CommonConfig.Flash == nil { + config.CommonConfig.Flash = &v1alpha1.Flash{} + } + if config.CommonConfig.Flash.FlashCluster == nil { + config.CommonConfig.Flash.FlashCluster = &v1alpha1.FlashCluster{} + } + if config.CommonConfig.Flash.FlashCluster.ClusterLog == "" { + config.CommonConfig.Flash.FlashCluster.ClusterLog = defaultClusterLog + } + if config.CommonConfig.Flash.FlashProxy == nil { + config.CommonConfig.Flash.FlashProxy = &v1alpha1.FlashProxy{} + } + if config.CommonConfig.Flash.FlashProxy.LogFile == "" { + config.CommonConfig.Flash.FlashProxy.LogFile = defaultProxyLog + } + + if config.CommonConfig.FlashLogger == nil { + config.CommonConfig.FlashLogger = &v1alpha1.FlashLogger{} + } + if config.CommonConfig.FlashLogger.ErrorLog == "" { + config.CommonConfig.FlashLogger.ErrorLog = defaultErrorLog + } + if config.CommonConfig.FlashLogger.ServerLog == "" { + config.CommonConfig.FlashLogger.ServerLog = defaultServerLog + } +} + +// setTiFlashConfigDefault sets default configs for TiFlash +func setTiFlashConfigDefault(config *v1alpha1.TiFlashConfig, clusterName, ns string) { + if config.CommonConfig == nil { + config.CommonConfig = &v1alpha1.CommonConfig{} + } + setTiFlashCommonConfigDefault(config.CommonConfig, clusterName, ns) + if config.ProxyConfig == nil { + config.ProxyConfig = &v1alpha1.ProxyConfig{} + } + setTiFlashProxyConfigDefault(config.ProxyConfig, clusterName, ns) +} + +func setTiFlashProxyConfigDefault(config *v1alpha1.ProxyConfig, clusterName, ns string) { + if config.LogLevel == "" { + config.LogLevel = "info" + } + if config.Server == nil { + config.Server = &v1alpha1.FlashServerConfig{} + } + if config.Server.EngineAddr == "" { + config.Server.EngineAddr = fmt.Sprintf("%s-POD_NUM.%s.%s.svc:3930", controller.TiFlashMemberName(clusterName), controller.TiFlashPeerMemberName(clusterName), ns) + } +} +func setTiFlashCommonConfigDefault(config *v1alpha1.CommonConfig, clusterName, ns string) { + if config.TmpPath == "" { + config.TmpPath = "/data0/tmp" + } + if config.DisplayName == "" { + config.DisplayName = "TiFlash" + } + if config.DefaultProfile == "" { + config.DefaultProfile = "default" + } + if config.Path == "" { + config.Path = "/data0/db" + } + if config.PathRealtimeMode == nil { + b := false + config.PathRealtimeMode = &b + } + if config.MarkCacheSize == nil { + var m int64 = 5368709120 + config.MarkCacheSize = &m + } + if config.MinmaxIndexCacheSize == nil { + var m int64 = 5368709120 + config.MinmaxIndexCacheSize = &m + } + if config.ListenHost == "" { + config.ListenHost = "0.0.0.0" + } + if config.TCPPort == nil { + var p int32 = 9000 + config.TCPPort = &p + } + if config.HTTPPort == nil { + var p int32 = 8123 + config.HTTPPort = &p + } + if config.InternalServerHTTPPort == nil { + var p int32 = 9009 + config.InternalServerHTTPPort = &p + } + if config.Flash == nil { + config.Flash = &v1alpha1.Flash{} + } + setTiFlashFlashConfigDefault(config.Flash, clusterName, ns) + if config.FlashLogger == nil { + config.FlashLogger = &v1alpha1.FlashLogger{} + } + setTiFlashLoggerConfigDefault(config.FlashLogger) + if config.FlashApplication == nil { + config.FlashApplication = &v1alpha1.FlashApplication{} + } + setTiFlashApplicationConfigDefault(config.FlashApplication) + if config.FlashRaft == nil { + config.FlashRaft = &v1alpha1.FlashRaft{} + } + setTiFlashRaftConfigDefault(config.FlashRaft, clusterName, ns) + if config.FlashStatus == nil { + config.FlashStatus = &v1alpha1.FlashStatus{} + } + setTiFlashStatusConfigDefault(config.FlashStatus) + if config.FlashQuota == nil { + config.FlashQuota = &v1alpha1.FlashQuota{} + } + setTiFlashQuotasConfigDefault(config.FlashQuota) + if config.FlashUser == nil { + config.FlashUser = &v1alpha1.FlashUser{} + } + setTiFlashUsersConfigDefault(config.FlashUser) + if config.FlashProfile == nil { + config.FlashProfile = &v1alpha1.FlashProfile{} + } + setTiFlashProfilesConfigDefault(config.FlashProfile) +} + +func setTiFlashFlashConfigDefault(config *v1alpha1.Flash, clusterName, ns string) { + if config.TiDBStatusAddr == "" { + config.TiDBStatusAddr = fmt.Sprintf("%s.%s.svc:10080", controller.TiDBMemberName(clusterName), ns) + } + if config.ServiceAddr == "" { + config.ServiceAddr = fmt.Sprintf("%s-POD_NUM.%s.%s.svc:3930", controller.TiFlashMemberName(clusterName), controller.TiFlashPeerMemberName(clusterName), ns) + } + if config.OverlapThreshold == nil { + o := 0.6 + config.OverlapThreshold = &o + } + if config.CompactLogMinPeriod == nil { + var o int32 = 200 + config.CompactLogMinPeriod = &o + } + if config.FlashCluster == nil { + config.FlashCluster = &v1alpha1.FlashCluster{} + } + setTiFlashFlashClusterConfigDefault(config.FlashCluster) + if config.FlashProxy == nil { + config.FlashProxy = &v1alpha1.FlashProxy{} + } + setTiFlashFlashProxyConfigDefault(config.FlashProxy, clusterName, ns) +} + +func setTiFlashFlashProxyConfigDefault(config *v1alpha1.FlashProxy, clusterName, ns string) { + if config.Addr == "" { + config.Addr = "0.0.0.0:20170" + } + if config.AdvertiseAddr == "" { + config.AdvertiseAddr = fmt.Sprintf("%s-POD_NUM.%s.%s.svc:20170", controller.TiFlashMemberName(clusterName), controller.TiFlashPeerMemberName(clusterName), ns) + } + if config.DataDir == "" { + config.DataDir = "/data0/proxy" + } + if config.Config == "" { + config.Config = "/data0/proxy.toml" + } + if config.LogFile == "" { + config.LogFile = defaultProxyLog + } +} + +func setTiFlashFlashClusterConfigDefault(config *v1alpha1.FlashCluster) { + if config.ClusterManagerPath == "" { + config.ClusterManagerPath = "/tiflash/flash_cluster_manager" + } + if config.ClusterLog == "" { + config.ClusterLog = defaultClusterLog + } + if config.RefreshInterval == nil { + var r int32 = 20 + config.RefreshInterval = &r + } + if config.UpdateRuleInterval == nil { + var r int32 = 10 + config.UpdateRuleInterval = &r + } + if config.MasterTTL == nil { + var r int32 = 60 + config.MasterTTL = &r + } +} + +func setTiFlashLoggerConfigDefault(config *v1alpha1.FlashLogger) { + if config.ErrorLog == "" { + config.ErrorLog = defaultErrorLog + } + if config.Size == "" { + config.Size = "100M" + } + if config.ServerLog == "" { + config.ServerLog = defaultServerLog + } + if config.Level == "" { + config.Level = "information" + } + if config.Count == nil { + var c int32 = 10 + config.Count = &c + } +} + +func setTiFlashApplicationConfigDefault(config *v1alpha1.FlashApplication) { + if config.RunAsDaemon == nil { + r := true + config.RunAsDaemon = &r + } +} + +func setTiFlashRaftConfigDefault(config *v1alpha1.FlashRaft, clusterName, ns string) { + if config.PDAddr == "" { + config.PDAddr = fmt.Sprintf("%s.%s.svc:2379", controller.PDMemberName(clusterName), ns) + } + if config.KVStorePath == "" { + config.KVStorePath = "/data0/kvstore" + } + if config.StorageEngine == "" { + config.StorageEngine = "dt" + } +} + +func setTiFlashStatusConfigDefault(config *v1alpha1.FlashStatus) { + if config.MetricsPort == nil { + var d int32 = 8234 + config.MetricsPort = &d + } +} + +func setTiFlashQuotasConfigDefault(config *v1alpha1.FlashQuota) { + if config.Default == nil { + config.Default = &v1alpha1.Quota{} + } + if config.Default.Interval == nil { + config.Default.Interval = &v1alpha1.Interval{} + } + if config.Default.Interval.Duration == nil { + var d int32 = 3600 + config.Default.Interval.Duration = &d + } + if config.Default.Interval.Queries == nil { + var d int32 = 0 + config.Default.Interval.Queries = &d + } + if config.Default.Interval.Errors == nil { + var d int32 = 0 + config.Default.Interval.Errors = &d + } + if config.Default.Interval.ResultRows == nil { + var d int32 = 0 + config.Default.Interval.ResultRows = &d + } + if config.Default.Interval.ReadRows == nil { + var d int32 = 0 + config.Default.Interval.ReadRows = &d + } + if config.Default.Interval.ExecutionTime == nil { + var d int32 = 0 + config.Default.Interval.ExecutionTime = &d + } +} + +func setTiFlashNetworksConfigDefault(config *v1alpha1.Networks) { + if config.IP == "" { + config.IP = "::/0" + } +} + +func setTiFlashUsersConfigDefault(config *v1alpha1.FlashUser) { + if config.Readonly == nil { + config.Readonly = &v1alpha1.User{} + } + if config.Readonly.Profile == "" { + config.Readonly.Profile = "readonly" + } + if config.Readonly.Quota == "" { + config.Readonly.Quota = "default" + } + if config.Readonly.Networks == nil { + config.Readonly.Networks = &v1alpha1.Networks{} + } + setTiFlashNetworksConfigDefault(config.Readonly.Networks) + + if config.Default == nil { + config.Default = &v1alpha1.User{} + } + if config.Default.Profile == "" { + config.Default.Profile = "default" + } + if config.Default.Quota == "" { + config.Default.Quota = "default" + } + if config.Default.Networks == nil { + config.Default.Networks = &v1alpha1.Networks{} + } + setTiFlashNetworksConfigDefault(config.Default.Networks) +} + +func setTiFlashProfilesConfigDefault(config *v1alpha1.FlashProfile) { + if config.Readonly == nil { + config.Readonly = &v1alpha1.Profile{} + } + if config.Readonly.Readonly == nil { + var r int32 = 1 + config.Readonly.Readonly = &r + } + if config.Default == nil { + config.Default = &v1alpha1.Profile{} + } + if config.Default.MaxMemoryUsage == nil { + var m int64 = 10000000000 + config.Default.MaxMemoryUsage = &m + } + if config.Default.UseUncompressedCache == nil { + var u int32 = 0 + config.Default.UseUncompressedCache = &u + } + if config.Default.LoadBalancing == nil { + l := "random" + config.Default.LoadBalancing = &l + } +} diff --git a/pkg/manager/member/tiflash_util_test.go b/pkg/manager/member/tiflash_util_test.go new file mode 100644 index 0000000000..032259be96 --- /dev/null +++ b/pkg/manager/member/tiflash_util_test.go @@ -0,0 +1,594 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "testing" + + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" +) + +var ( + defaultTiFlashConfig = v1alpha1.TiFlashConfig{ + CommonConfig: &v1alpha1.CommonConfig{ + FlashApplication: &v1alpha1.FlashApplication{ + RunAsDaemon: pointer.BoolPtr(true), + }, + DefaultProfile: "default", + DisplayName: "TiFlash", + Flash: &v1alpha1.Flash{ + CompactLogMinPeriod: pointer.Int32Ptr(200), + FlashCluster: &v1alpha1.FlashCluster{ + ClusterManagerPath: "/tiflash/flash_cluster_manager", + ClusterLog: "/data0/logs/flash_cluster_manager.log", + MasterTTL: pointer.Int32Ptr(60), + RefreshInterval: pointer.Int32Ptr(20), + UpdateRuleInterval: pointer.Int32Ptr(10), + }, + OverlapThreshold: pointer.Float64Ptr(0.6), + FlashProxy: &v1alpha1.FlashProxy{ + Addr: "0.0.0.0:20170", + AdvertiseAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:20170", + Config: "/data0/proxy.toml", + DataDir: "/data0/proxy", + LogFile: "/data0/logs/proxy.log", + }, + ServiceAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:3930", + TiDBStatusAddr: "test-tidb.test.svc:10080", + }, + HTTPPort: pointer.Int32Ptr(8123), + InternalServerHTTPPort: pointer.Int32Ptr(9009), + ListenHost: "0.0.0.0", + FlashLogger: &v1alpha1.FlashLogger{ + Count: pointer.Int32Ptr(10), + ErrorLog: "/data0/logs/error.log", + Level: "information", + ServerLog: "/data0/logs/server.log", + Size: "100M", + }, + MarkCacheSize: pointer.Int64Ptr(5368709120), + MinmaxIndexCacheSize: pointer.Int64Ptr(5368709120), + Path: "/data0/db", + PathRealtimeMode: pointer.BoolPtr(false), + FlashProfile: &v1alpha1.FlashProfile{ + Default: &v1alpha1.Profile{ + LoadBalancing: pointer.StringPtr("random"), + MaxMemoryUsage: pointer.Int64Ptr(10000000000), + UseUncompressedCache: pointer.Int32Ptr(0), + }, + Readonly: &v1alpha1.Profile{ + Readonly: pointer.Int32Ptr(1), + }, + }, + FlashQuota: &v1alpha1.FlashQuota{ + Default: &v1alpha1.Quota{ + Interval: &v1alpha1.Interval{ + Duration: pointer.Int32Ptr(3600), + Errors: pointer.Int32Ptr(0), + ExecutionTime: pointer.Int32Ptr(0), + Queries: pointer.Int32Ptr(0), + ReadRows: pointer.Int32Ptr(0), + ResultRows: pointer.Int32Ptr(0), + }, + }, + }, + FlashRaft: &v1alpha1.FlashRaft{ + KVStorePath: "/data0/kvstore", + PDAddr: "test-pd.test.svc:2379", + StorageEngine: "dt", + }, + FlashStatus: &v1alpha1.FlashStatus{ + MetricsPort: pointer.Int32Ptr(8234), + }, + TCPPort: pointer.Int32Ptr(9000), + TmpPath: "/data0/tmp", + FlashUser: &v1alpha1.FlashUser{ + Default: &v1alpha1.User{ + Networks: &v1alpha1.Networks{ + IP: "::/0", + }, + Profile: "default", + Quota: "default", + }, + Readonly: &v1alpha1.User{ + Networks: &v1alpha1.Networks{ + IP: "::/0", + }, + Profile: "readonly", + Quota: "default", + }, + }, + }, + ProxyConfig: &v1alpha1.ProxyConfig{ + LogLevel: "info", + Server: &v1alpha1.FlashServerConfig{ + EngineAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:3930", + }, + }, + } + customTiFlashConfig = v1alpha1.TiFlashConfig{ + CommonConfig: &v1alpha1.CommonConfig{ + FlashApplication: &v1alpha1.FlashApplication{ + RunAsDaemon: pointer.BoolPtr(false), + }, + DefaultProfile: "defaul", + DisplayName: "TiFlah", + Flash: &v1alpha1.Flash{ + CompactLogMinPeriod: pointer.Int32Ptr(100), + FlashCluster: &v1alpha1.FlashCluster{ + ClusterManagerPath: "/flash_cluster_manager", + ClusterLog: "/data1/logs/flash_cluster_manager.log", + MasterTTL: pointer.Int32Ptr(50), + RefreshInterval: pointer.Int32Ptr(21), + UpdateRuleInterval: pointer.Int32Ptr(11), + }, + OverlapThreshold: pointer.Float64Ptr(0.7), + FlashProxy: &v1alpha1.FlashProxy{ + Addr: "0.0.0.0:20171", + AdvertiseAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:20171", + Config: "/data0/proxy1.toml", + DataDir: "/data0/proxy1", + LogFile: "/data0/logs/proxy1.log", + }, + ServiceAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:3931", + TiDBStatusAddr: "test-tidb.test.svc:10081", + }, + HTTPPort: pointer.Int32Ptr(8121), + InternalServerHTTPPort: pointer.Int32Ptr(9001), + ListenHost: "0.0.0.1", + FlashLogger: &v1alpha1.FlashLogger{ + Count: pointer.Int32Ptr(11), + ErrorLog: "/data1/logs/error1.log", + Level: "information1", + ServerLog: "/data0/logs/server1.log", + Size: "101M", + }, + MarkCacheSize: pointer.Int64Ptr(5368709121), + MinmaxIndexCacheSize: pointer.Int64Ptr(5368709121), + Path: "/data1/db", + PathRealtimeMode: pointer.BoolPtr(true), + FlashProfile: &v1alpha1.FlashProfile{ + Default: &v1alpha1.Profile{ + LoadBalancing: pointer.StringPtr("random1"), + MaxMemoryUsage: pointer.Int64Ptr(10000000001), + UseUncompressedCache: pointer.Int32Ptr(1), + }, + Readonly: &v1alpha1.Profile{ + Readonly: pointer.Int32Ptr(0), + }, + }, + FlashQuota: &v1alpha1.FlashQuota{ + Default: &v1alpha1.Quota{ + Interval: &v1alpha1.Interval{ + Duration: pointer.Int32Ptr(3601), + Errors: pointer.Int32Ptr(1), + ExecutionTime: pointer.Int32Ptr(1), + Queries: pointer.Int32Ptr(1), + ReadRows: pointer.Int32Ptr(1), + ResultRows: pointer.Int32Ptr(1), + }, + }, + }, + FlashRaft: &v1alpha1.FlashRaft{ + KVStorePath: "/data1/kvstore", + PDAddr: "test-pd.test.svc:2379", + StorageEngine: "dt", + }, + FlashStatus: &v1alpha1.FlashStatus{ + MetricsPort: pointer.Int32Ptr(8235), + }, + TCPPort: pointer.Int32Ptr(9001), + TmpPath: "/data1/tmp", + FlashUser: &v1alpha1.FlashUser{ + Default: &v1alpha1.User{ + Networks: &v1alpha1.Networks{ + IP: "::/1", + }, + Profile: "default1", + Quota: "default1", + }, + Readonly: &v1alpha1.User{ + Networks: &v1alpha1.Networks{ + IP: "::/1", + }, + Profile: "readonly1", + Quota: "default1", + }, + }, + }, + ProxyConfig: &v1alpha1.ProxyConfig{ + LogLevel: "info1", + Server: &v1alpha1.FlashServerConfig{ + EngineAddr: "test-tiflash-POD_NUM.test-tiflash-peer.test.svc:3930", + }, + }, + } + defaultTiFlashLogConfig = v1alpha1.TiFlashConfig{ + CommonConfig: &v1alpha1.CommonConfig{ + Flash: &v1alpha1.Flash{ + FlashCluster: &v1alpha1.FlashCluster{ + ClusterLog: "/data0/logs/flash_cluster_manager.log", + }, + FlashProxy: &v1alpha1.FlashProxy{ + LogFile: "/data0/logs/proxy.log", + }, + }, + FlashLogger: &v1alpha1.FlashLogger{ + ErrorLog: "/data0/logs/error.log", + ServerLog: "/data0/logs/server.log", + }, + }, + } + customTiFlashLogConfig = v1alpha1.TiFlashConfig{ + CommonConfig: &v1alpha1.CommonConfig{ + Flash: &v1alpha1.Flash{ + FlashCluster: &v1alpha1.FlashCluster{ + ClusterLog: "/data1/logs/flash_cluster_manager.log", + }, + FlashProxy: &v1alpha1.FlashProxy{ + LogFile: "/data1/logs/proxy.log", + }, + }, + FlashLogger: &v1alpha1.FlashLogger{ + ErrorLog: "/data1/logs/error.log", + ServerLog: "/data1/logs/server.log", + }, + }, + } + defaultSideCarContainers = []corev1.Container{ + { + Name: "serverlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data0/logs/server.log; tail -n0 -F /data0/logs/server.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + }, + }, + { + Name: "errorlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data0/logs/error.log; tail -n0 -F /data0/logs/error.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + }, + }, + { + Name: "proxylog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data0/logs/proxy.log; tail -n0 -F /data0/logs/proxy.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + }, + }, + { + Name: "clusterlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data0/logs/flash_cluster_manager.log; tail -n0 -F /data0/logs/flash_cluster_manager.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + }, + }, + } + customSideCarContainers = []corev1.Container{ + { + Name: "serverlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/server.log; tail -n0 -F /data1/logs/server.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "errorlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/error.log; tail -n0 -F /data1/logs/error.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "proxylog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/proxy.log; tail -n0 -F /data1/logs/proxy.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "clusterlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{}, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/flash_cluster_manager.log; tail -n0 -F /data1/logs/flash_cluster_manager.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + } + customResourceSideCarContainers = []corev1.Container{ + { + Name: "serverlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/server.log; tail -n0 -F /data1/logs/server.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "errorlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/error.log; tail -n0 -F /data1/logs/error.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "proxylog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/proxy.log; tail -n0 -F /data1/logs/proxy.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + { + Name: "clusterlog", + Image: "busybox:1.26.2", + ImagePullPolicy: "", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + Command: []string{ + "sh", + "-c", + "touch /data1/logs/flash_cluster_manager.log; tail -n0 -F /data1/logs/flash_cluster_manager.log;", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "data1", MountPath: "/data1"}, + }, + }, + } +) + +func newTidbCluster() *v1alpha1.TidbCluster { + return &v1alpha1.TidbCluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "TidbCluster", + APIVersion: "pingcap.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pd", + Namespace: corev1.NamespaceDefault, + UID: types.UID("test"), + }, + Spec: v1alpha1.TidbClusterSpec{ + PD: v1alpha1.PDSpec{ + ComponentSpec: v1alpha1.ComponentSpec{ + Image: "pd-test-image", + }, + }, + TiKV: v1alpha1.TiKVSpec{ + ComponentSpec: v1alpha1.ComponentSpec{ + Image: "tikv-test-image", + }, + }, + TiDB: v1alpha1.TiDBSpec{ + ComponentSpec: v1alpha1.ComponentSpec{ + Image: "tidb-test-image", + }, + }, + TiFlash: &v1alpha1.TiFlashSpec{}, + }, + } +} + +func TestBuildTiFlashSidecarContainers(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + flashConfig *v1alpha1.TiFlashConfig + expect []corev1.Container + resource bool + } + + tests := []*testcase{ + { + name: "nil config", + flashConfig: nil, + expect: defaultSideCarContainers, + }, + { + name: "empty config", + flashConfig: &v1alpha1.TiFlashConfig{}, + expect: defaultSideCarContainers, + }, + { + name: "custom config", + flashConfig: &customTiFlashLogConfig, + expect: customSideCarContainers, + }, + { + name: "custom resource config", + flashConfig: &customTiFlashLogConfig, + expect: customResourceSideCarContainers, + resource: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tc := newTidbCluster() + tc.Spec.TiFlash.Config = test.flashConfig + if test.resource { + tc.Spec.TiFlash.LogTailer = &v1alpha1.LogTailerSpec{} + tc.Spec.TiFlash.LogTailer.ResourceRequirements = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + corev1.ResourceStorage: resource.MustParse("100Gi"), + }, + } + } + cs := buildTiFlashSidecarContainers(tc) + g.Expect(cs).To(Equal(test.expect)) + }) + } +} +func TestSetTiFlashConfigDefault(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + config v1alpha1.TiFlashConfig + expect v1alpha1.TiFlashConfig + } + + tests := []*testcase{ + { + name: "nil config", + config: v1alpha1.TiFlashConfig{}, + expect: defaultTiFlashConfig, + }, + { + name: "custom config", + config: customTiFlashConfig, + expect: customTiFlashConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + setTiFlashConfigDefault(&test.config, "test", "test") + g.Expect(test.config).To(Equal(test.expect)) + }) + } +} + +func TestSetTiFlashLogConfigDefault(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + config v1alpha1.TiFlashConfig + expect v1alpha1.TiFlashConfig + } + + tests := []*testcase{ + { + name: "nil config", + config: v1alpha1.TiFlashConfig{}, + expect: defaultTiFlashLogConfig, + }, + { + name: "custom config", + config: customTiFlashLogConfig, + expect: customTiFlashLogConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + setTiFlashLogConfigDefault(&test.config) + g.Expect(test.config).To(Equal(test.expect)) + }) + } +}