Support the use of kubeadm to manage etcd and use of existing etcd

Signed-off-by: pixiake <guofeng@yunify.com>

apis/kubekey/v1alpha2/cluster_types.go

@@ -44,6 +44,7 @@ type ClusterSpec struct {
 	RoleGroups           map[string][]string  `yaml:"roleGroups" json:"roleGroups,omitempty"`
 	ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"`
 	System               System               `yaml:"system" json:"system,omitempty"`
+	Etcd                 EtcdCluster          `yaml:"etcd" json:"etcd,omitempty"`
 	Kubernetes           Kubernetes           `yaml:"kubernetes" json:"kubernetes,omitempty"`
 	Network              NetworkConfig        `yaml:"network" json:"network,omitempty"`
 	Registry             RegistryConfig       `yaml:"registry" json:"registry,omitempty"`
@@ -186,14 +187,6 @@ type KubeSphere struct {
 	Configurations string `json:"configurations,omitempty"`
 }
 
-// ExternalEtcd defines configuration information of external etcd.
-type ExternalEtcd struct {
-	Endpoints []string
-	CaFile    string
-	CertFile  string
-	KeyFile   string
-}
-
 // GenerateCertSANs is used to generate cert sans for cluster.
 func (cfg *ClusterSpec) GenerateCertSANs() []string {
 	clusterSvc := fmt.Sprintf("kubernetes.default.svc.%s", cfg.Kubernetes.DNSDomain)
@@ -239,7 +232,7 @@ func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost {
 	if len(roleGroups[Master]) == 0 && len(roleGroups[ControlPlane]) == 0 {
 		logger.Log.Fatal(errors.New("The number of master/control-plane cannot be 0"))
 	}
-	if len(roleGroups[Etcd]) == 0 {
+	if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey {
 		logger.Log.Fatal(errors.New("The number of etcd cannot be 0"))
 	}
 	if len(roleGroups[Registry]) > 1 {

apis/kubekey/v1alpha2/default.go

@@ -92,6 +92,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool) (*ClusterSpec, map
 
 	clusterCfg.Hosts = SetDefaultHostsCfg(cfg)
 	clusterCfg.RoleGroups = cfg.RoleGroups
+	clusterCfg.Etcd = SetDefaultEtcdCfg(cfg)
 	roleGroups := clusterCfg.GroupHosts()
 	clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, roleGroups[Master], incluster)
 	clusterCfg.Network = SetDefaultNetworkCfg(cfg)
@@ -254,18 +255,6 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.DNSDomain == "" {
 		cfg.Kubernetes.DNSDomain = DefaultDNSDomain
 	}
-	if cfg.Kubernetes.EtcdBackupDir == "" {
-		cfg.Kubernetes.EtcdBackupDir = DefaultEtcdBackupDir
-	}
-	if cfg.Kubernetes.EtcdBackupPeriod == 0 {
-		cfg.Kubernetes.EtcdBackupPeriod = DefaultEtcdBackupPeriod
-	}
-	if cfg.Kubernetes.KeepBackupNumber == 0 {
-		cfg.Kubernetes.KeepBackupNumber = DefaultKeepBackNumber
-	}
-	if cfg.Kubernetes.EtcdBackupScriptDir == "" {
-		cfg.Kubernetes.EtcdBackupScriptDir = DefaultEtcdBackupScriptDir
-	}
 	if cfg.Kubernetes.ContainerManager == "" {
 		cfg.Kubernetes.ContainerManager = Docker
 	}
@@ -287,3 +276,23 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 
 	return defaultClusterCfg
 }
+
+func SetDefaultEtcdCfg(cfg *ClusterSpec) EtcdCluster {
+	if cfg.Etcd.Type == "" || ((cfg.Kubernetes.Type == "k3s" || (len(strings.Split(cfg.Kubernetes.Version, "-")) > 1) && strings.Split(cfg.Kubernetes.Version, "-")[1] == "k3s") && cfg.Etcd.Type == Kubeadm) {
+		cfg.Etcd.Type = KubeKey
+	}
+	if cfg.Etcd.BackupDir == "" {
+		cfg.Etcd.BackupDir = DefaultEtcdBackupDir
+	}
+	if cfg.Etcd.BackupPeriod == 0 {
+		cfg.Etcd.BackupPeriod = DefaultEtcdBackupPeriod
+	}
+	if cfg.Etcd.KeepBackupNumber == 0 {
+		cfg.Etcd.KeepBackupNumber = DefaultKeepBackNumber
+	}
+	if cfg.Etcd.BackupScriptDir == "" {
+		cfg.Etcd.BackupScriptDir = DefaultEtcdBackupScriptDir
+	}
+
+	return cfg.Etcd
+}

apis/kubekey/v1alpha2/etcd_types.go

@@ -0,0 +1,48 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package v1alpha2
+
+const (
+	KubeKey  = "kubekey"
+	Kubeadm  = "kubeadm"
+	External = "external"
+)
+
+type EtcdCluster struct {
+	// Type of etcd cluster, can be set to 'kubekey' 'kubeadm' 'external'
+	Type string `yaml:"type" json:"type,omitempty"`
+	// ExternalEtcd describes how to connect to an external etcd cluster when type is set to external
+	External         ExternalEtcd `yaml:"external" json:"external,omitempty"`
+	BackupDir        string       `yaml:"backupDir" json:"backupDir,omitempty"`
+	BackupPeriod     int          `yaml:"backupPeriod" json:"backupPeriod,omitempty"`
+	KeepBackupNumber int          `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"`
+	BackupScriptDir  string       `yaml:"backupScript" json:"backupScript,omitempty"`
+}
+
+// ExternalEtcd describes how to connect to an external etcd cluster
+// KubeKey, Kubeadm and External are mutually exclusive
+type ExternalEtcd struct {
+	// Endpoints of etcd members. Useful for using external etcd.
+	// If not provided, kubeadm will run etcd in a static pod.
+	Endpoints []string `yaml:"endpoints" json:"endpoints,omitempty"`
+	// CAFile is an SSL Certificate Authority file used to secure etcd communication.
+	CAFile string `yaml:"caFile" json:"caFile,omitempty"`
+	// CertFile is an SSL certification file used to secure etcd communication.
+	CertFile string `yaml:"certFile" json:"certFile,omitempty"`
+	// KeyFile is an SSL key file used to secure etcd communication.
+	KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"`
+}

apis/kubekey/v1alpha2/kubernetes_types.go

@@ -31,10 +31,6 @@ type Kubernetes struct {
 	ProxyMode              string   `yaml:"proxyMode" json:"proxyMode,omitempty"`
 	// +optional
 	Nodelocaldns             *bool                `yaml:"nodelocaldns" json:"nodelocaldns,omitempty"`
-	EtcdBackupDir            string               `yaml:"etcdBackupDir" json:"etcdBackupDir,omitempty"`
-	EtcdBackupPeriod         int                  `yaml:"etcdBackupPeriod" json:"etcdBackupPeriod,omitempty"`
-	KeepBackupNumber         int                  `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"`
-	EtcdBackupScriptDir      string               `yaml:"etcdBackupScript" json:"etcdBackupScript,omitempty"`
 	ContainerManager         string               `yaml:"containerManager" json:"containerManager,omitempty"`
 	ContainerRuntimeEndpoint string               `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"`
 	NodeFeatureDiscovery     NodeFeatureDiscovery `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"`

cmd/ctl/create/cluster.go

@@ -42,7 +42,6 @@ type CreateClusterOptions struct {
 	DownloadCmd      string
 	Artifact         string
 	InstallPackages  bool
-	CertificatesDir  string
 
 	localStorageChanged bool
 }
@@ -118,7 +117,6 @@ func (o *CreateClusterOptions) Run() error {
 		ContainerManager:  o.ContainerManager,
 		Artifact:          o.Artifact,
 		InstallPackages:   o.InstallPackages,
-		CertificatesDir:   o.CertificatesDir,
 	}
 
 	if o.localStorageChanged {
@@ -137,7 +135,6 @@ func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images")
 	cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images")
 	cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.")
-	cmd.Flags().StringVarP(&o.CertificatesDir, "certificates-dir", "", "", "Specifies where to store or look for all required certificates.")
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")

config/crd/bases/kubekey.kubesphere.io_clusters.yaml

@@ -430,6 +430,45 @@ spec:
                   port:
                     type: integer
                 type: object
+              etcd:
+                properties:
+                  backupDir:
+                    type: string
+                  backupPeriod:
+                    type: integer
+                  backupScript:
+                    type: string
+                  external:
+                    description: ExternalEtcd describes how to connect to an external
+                      etcd cluster when type is set to external
+                    properties:
+                      caFile:
+                        description: CAFile is an SSL Certificate Authority file used
+                          to secure etcd communication.
+                        type: string
+                      certFile:
+                        description: CertFile is an SSL certification file used to
+                          secure etcd communication.
+                        type: string
+                      endpoints:
+                        description: Endpoints of etcd members. Useful for using external
+                          etcd. If not provided, kubeadm will run etcd in a static
+                          pod.
+                        items:
+                          type: string
+                        type: array
+                      keyFile:
+                        description: KeyFile is an SSL key file used to secure etcd
+                          communication.
+                        type: string
+                    type: object
+                  keepBackupNumber:
+                    type: integer
+                  type:
+                    description: Type of etcd cluster, can be set to 'kubekey' 'kubeadm'
+                      'external'
+                    type: string
+                type: object
               hosts:
                 description: Foo is an example field of Cluster. Edit Cluster_types.go
                   to remove/update
@@ -440,13 +479,12 @@ spec:
                       type: string
                     arch:
                       type: string
-                    id:
-                      type: string
                     internalAddress:
                       type: string
                     labels:
                       additionalProperties:
                         type: string
+                      description: Labels defines the kubernetes labels for the node.
                       type: object
                     name:
                       type: string
@@ -485,25 +523,16 @@ spec:
                     type: array
                   dnsDomain:
                     type: string
-                  etcdBackupDir:
-                    type: string
-                  etcdBackupPeriod:
-                    type: integer
-                  etcdBackupScript:
-                    type: string
                   featureGates:
                     additionalProperties:
                       type: boolean
                     type: object
                   kata:
-                    description: NodeFeatureDiscovery contains the configuration for
-                      the node-feature-discovery in cluster
+                    description: Kata contains the configuration for the kata in cluster
                     properties:
                       enabled:
                         type: boolean
                     type: object
-                  keepBackupNumber:
-                    type: integer
                   kubeProxyArgs:
                     items:
                       type: string
@@ -523,7 +552,8 @@ spec:
                   nodeCidrMaskSize:
                     type: integer
                   nodeFeatureDiscovery:
-                    description: Kata contains the configuration for the kata in cluster
+                    description: NodeFeatureDiscovery contains the configuration for
+                      the node-feature-discovery in cluster
                     properties:
                       enabled:
                         type: boolean
@@ -563,8 +593,6 @@ spec:
                       vxlanMode:
                         type: string
                     type: object
-                  enableMultusCNI:
-                    type: boolean
                   flannel:
                     properties:
                       backendMode:
@@ -605,19 +633,26 @@ spec:
                       vlanInterfaceName:
                         type: string
                     type: object
+                  multusCNI:
+                    properties:
+                      enabled:
+                        type: boolean
+                    type: object
                   plugin:
                     type: string
                 type: object
               registry:
                 description: RegistryConfig defines the configuration information
                   of the image's repository.
                 properties:
-                  Auths:
+                  auths:
                     type: object
                   insecureRegistries:
                     items:
                       type: string
                     type: array
+                  namespaceOverride:
+                    type: string
                   plainHTTP:
                     type: boolean
                   privateRegistry:

docs/config-example.md

@@ -40,6 +40,16 @@ spec:
       ExpandCSIVolumes: true
       RotateKubeletServerCertificate: true
       TTLAfterFinished: true
+  etcd:
+    type: kubekey  # Specify the type of etcd used by the cluster. When the cluster type is k3s, setting this parameter to kubeadm is invalid. [kubekey | kubeadm | external] [Default: kubekey]
+    ## The following parameters need to be added only when the type is set to external.
+    ## caFile, certFile and keyFile need not be set, if TLS authentication is not enabled for the existing etcd.
+    # external:
+    #   endpoints:
+    #     - https://192.168.6.6:2379
+    #   caFile: /pki/etcd/ca.crt
+    #   certFile: /pki/etcd/etcd.crt
+    #   keyFile: /pki/etcd/etcd.key
   network:
     plugin: calico
     calico:

pkg/bootstrap/os/module.go

@@ -95,11 +95,12 @@ func (c *ClearOSEnvironmentModule) Init() {
 		Parallel: true,
 	}
 
-	stopETCD := &task.RemoteTask{
-		Name:     "StopETCDService",
-		Desc:     "Stop etcd service",
+	uninstallETCD := &task.RemoteTask{
+		Name:     "UninstallETCD",
+		Desc:     "Uninstall etcd",
 		Hosts:    c.Runtime.GetHostsByRole(common.ETCD),
-		Action:   new(StopETCDService),
+		Prepare:  new(EtcdTypeIsKubeKey),
+		Action:   new(UninstallETCD),
 		Parallel: true,
 	}
 
@@ -121,7 +122,7 @@ func (c *ClearOSEnvironmentModule) Init() {
 
 	c.Tasks = []task.Interface{
 		resetNetworkConfig,
-		stopETCD,
+		uninstallETCD,
 		removeFiles,
 		daemonReload,
 	}

pkg/bootstrap/os/prepares.go

@@ -17,6 +17,7 @@
 package os
 
 import (
+	kubekeyv1alpha2 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/pkg/common"
 	"github.com/kubesphere/kubekey/pkg/core/connector"
 )
@@ -33,3 +34,15 @@ func (n *NodeConfigureNtpCheck) PreCheck(_ connector.Runtime) (bool, error) {
 
 	return true, nil
 }
+
+type EtcdTypeIsKubeKey struct {
+	common.KubePrepare
+}
+
+func (e *EtcdTypeIsKubeKey) PreCheck(_ connector.Runtime) (bool, error) {
+	if len(e.KubeConf.Cluster.Etcd.Type) == 0 || e.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.KubeKey {
+		return true, nil
+	}
+
+	return false, nil
+}