[release-1.21] Add etcd extra args support for K3s (#4471)

* Export cli server flags and etcd restoration functions (#3527) * Export cli server flags and etfd restoration functions Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * export S3 Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> Signed-off-by: Chris Kim <oats87g@gmail.com> * Add etcd extra args support for K3s Signed-off-by: Chris Kim <oats87g@gmail.com> * Remove integration test Signed-off-by: Chris Kim <oats87g@gmail.com> Co-authored-by: Hussein Galal <galal-hussein@users.noreply.github.com>
k3s-io · Nov 12, 2021 · 334eae1 · 334eae1
1 parent 10c854c
commit 334eae1
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 57 deletions.
diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go
@@ -38,6 +38,7 @@ type Server struct {
 	TLSSan                   cli.StringSlice
 	BindAddress              string
 	ExtraAPIArgs             cli.StringSlice
+	ExtraEtcdArgs            cli.StringSlice
 	ExtraSchedulerArgs       cli.StringSlice
 	ExtraControllerArgs      cli.StringSlice
 	ExtraCloudControllerArgs cli.StringSlice
@@ -83,7 +84,56 @@ type Server struct {
 	EtcdS3Insecure           bool
 }
 
-var ServerConfig Server
+var (
+	ServerConfig Server
+	ClusterCIDR  = cli.StringSliceFlag{
+		Name:  "cluster-cidr",
+		Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
+		Value: &ServerConfig.ClusterCIDR,
+	}
+	ServiceCIDR = cli.StringSliceFlag{
+		Name:  "service-cidr",
+		Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)",
+		Value: &ServerConfig.ServiceCIDR,
+	}
+	ServiceNodePortRange = cli.StringFlag{
+		Name:        "service-node-port-range",
+		Usage:       "(networking) Port range to reserve for services with NodePort visibility",
+		Destination: &ServerConfig.ServiceNodePortRange,
+		Value:       "30000-32767",
+	}
+	ClusterDNS = cli.StringSliceFlag{
+		Name:  "cluster-dns",
+		Usage: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
+		Value: &ServerConfig.ClusterDNS,
+	}
+	ClusterDomain = cli.StringFlag{
+		Name:        "cluster-domain",
+		Usage:       "(networking) Cluster Domain",
+		Destination: &ServerConfig.ClusterDomain,
+		Value:       "cluster.local",
+	}
+	ExtraAPIArgs = cli.StringSliceFlag{
+		Name:  "kube-apiserver-arg",
+		Usage: "(flags) Customized flag for kube-apiserver process",
+		Value: &ServerConfig.ExtraAPIArgs,
+	}
+	ExtraEtcdArgs = cli.StringSliceFlag{
+		Name:  "etcd-arg",
+		Usage: "(flags) Customized flag for etcd process",
+		Value: &ServerConfig.ExtraEtcdArgs,
+	}
+	ExtraSchedulerArgs = cli.StringSliceFlag{
+		Name:  "kube-scheduler-arg",
+		Usage: "(flags) Customized flag for kube-scheduler process",
+		Value: &ServerConfig.ExtraSchedulerArgs,
+	}
+	ExtraControllerArgs = cli.StringSliceFlag{
+		Name:  "kube-controller-manager-arg",
+		Usage: "(flags) Customized flag for kube-controller-manager process",
+		Value: &ServerConfig.ExtraControllerArgs,
+	}
+)
 
 func NewServerCommand(action func(*cli.Context) error) cli.Command {
 	return cli.Command{
@@ -130,33 +180,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
 				Destination: &ServerConfig.DataDir,
 			},
-			cli.StringSliceFlag{
-				Name:  "cluster-cidr",
-				Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
-				Value: &ServerConfig.ClusterCIDR,
-			},
-			cli.StringSliceFlag{
-				Name:  "service-cidr",
-				Usage: "(networking) IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16)",
-				Value: &ServerConfig.ServiceCIDR,
-			},
-			cli.StringFlag{
-				Name:        "service-node-port-range",
-				Usage:       "(networking) Port range to reserve for services with NodePort visibility",
-				Destination: &ServerConfig.ServiceNodePortRange,
-				Value:       "30000-32767",
-			},
-			cli.StringSliceFlag{
-				Name:  "cluster-dns",
-				Usage: "(networking) IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
-				Value: &ServerConfig.ClusterDNS,
-			},
-			cli.StringFlag{
-				Name:        "cluster-domain",
-				Usage:       "(networking) Cluster Domain",
-				Destination: &ServerConfig.ClusterDomain,
-				Value:       "cluster.local",
-			},
+			ClusterCIDR,
+			ServiceCIDR,
+			ServiceNodePortRange,
+			ClusterDNS,
+			ClusterDomain,
 			cli.StringFlag{
 				Name:        "flannel-backend",
 				Usage:       "(networking) One of 'none', 'vxlan', 'ipsec', 'host-gw', or 'wireguard'",
@@ -187,21 +215,10 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.KubeConfigMode,
 				EnvVar:      version.ProgramUpper + "_KUBECONFIG_MODE",
 			},
-			cli.StringSliceFlag{
-				Name:  "kube-apiserver-arg",
-				Usage: "(flags) Customized flag for kube-apiserver process",
-				Value: &ServerConfig.ExtraAPIArgs,
-			},
-			cli.StringSliceFlag{
-				Name:  "kube-scheduler-arg",
-				Usage: "(flags) Customized flag for kube-scheduler process",
-				Value: &ServerConfig.ExtraSchedulerArgs,
-			},
-			cli.StringSliceFlag{
-				Name:  "kube-controller-manager-arg",
-				Usage: "(flags) Customized flag for kube-controller-manager process",
-				Value: &ServerConfig.ExtraControllerArgs,
-			},
+			ExtraAPIArgs,
+			ExtraEtcdArgs,
+			ExtraControllerArgs,
+			ExtraSchedulerArgs,
 			cli.StringSliceFlag{
 				Name:  "kube-cloud-controller-manager-arg",
 				Usage: "(flags) Customized flag for kube-cloud-controller-manager process",

diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go
@@ -117,6 +117,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
+	serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs
 	serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
 	serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain
 	serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint

diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go
@@ -74,7 +74,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error {
 					ElectionTimeout:   5000,
 					LogOutputs:        []string{"stderr"},
 				}
-				configFile, err := args.ToConfigFile()
+				configFile, err := args.ToConfigFile(c.config.ExtraEtcdArgs)
 				if err != nil {
 					return err
 				}

diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go
@@ -132,6 +132,7 @@ type Control struct {
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string
+	ExtraEtcdArgs            []string
 	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
 	JoinURL                  string

diff --git a/pkg/daemons/executor/etcd.go b/pkg/daemons/executor/etcd.go
@@ -18,8 +18,8 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) {
 	return InitialOptions{}, nil
 }
 
-func (e Embedded) ETCD(args ETCDConfig) error {
-	configFile, err := args.ToConfigFile()
+func (e Embedded) ETCD(args ETCDConfig, extraArgs []string) error {
+	configFile, err := args.ToConfigFile(extraArgs)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go
@@ -6,12 +6,15 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-
-	"sigs.k8s.io/yaml"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
+	yaml2 "gopkg.in/yaml.v2"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"sigs.k8s.io/yaml"
 )
 
 var (
@@ -27,7 +30,7 @@ type Executor interface {
 	Scheduler(apiReady <-chan struct{}, args []string) error
 	ControllerManager(apiReady <-chan struct{}, args []string) error
 	CurrentETCDOptions() (InitialOptions, error)
-	ETCD(args ETCDConfig) error
+	ETCD(args ETCDConfig, extraArgs []string) error
 	CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error
 }
 
@@ -69,13 +72,53 @@ type InitialOptions struct {
 	State            string `json:"initial-cluster-state,omitempty"`
 }
 
-func (e ETCDConfig) ToConfigFile() (string, error) {
+func (e ETCDConfig) ToConfigFile(extraArgs []string) (string, error) {
 	confFile := filepath.Join(e.DataDir, "config")
 	bytes, err := yaml.Marshal(&e)
 	if err != nil {
 		return "", err
 	}
 
+	if len(extraArgs) > 0 {
+		var s map[string]interface{}
+		if err := yaml2.Unmarshal(bytes, &s); err != nil {
+			return "", err
+		}
+
+		for _, v := range extraArgs {
+			extraArg := strings.SplitN(v, "=", 2)
+			// Depending on the argV, we have different types to handle.
+			// Source: https://github.com/etcd-io/etcd/blob/44b8ae145b505811775f5af915dd19198d556d55/server/config/config.go#L36-L190 and https://etcd.io/docs/v3.5/op-guide/configuration/#configuration-file
+			if len(extraArg) == 2 {
+				key := strings.TrimLeft(extraArg[0], "-")
+				lowerKey := strings.ToLower(key)
+				var stringArr []string
+				if i, err := strconv.Atoi(extraArg[1]); err == nil {
+					s[key] = i
+				} else if time, err := time.ParseDuration(extraArg[1]); err == nil && (strings.Contains(lowerKey, "time") || strings.Contains(lowerKey, "duration") || strings.Contains(lowerKey, "interval") || strings.Contains(lowerKey, "retention")) {
+					// auto-compaction-retention is either a time.Duration or int, depending on version. If it is an int, it will be caught above.
+					s[key] = time
+				} else if err := yaml.Unmarshal([]byte(extraArg[1]), &stringArr); err == nil {
+					s[key] = stringArr
+				} else {
+					switch strings.ToLower(extraArg[1]) {
+					case "true":
+						s[key] = true
+					case "false":
+						s[key] = false
+					default:
+						s[key] = extraArg[1]
+					}
+				}
+			}
+		}
+
+		bytes, err = yaml2.Marshal(&s)
+		if err != nil {
+			return "", err
+		}
+	}
+
 	if err := os.MkdirAll(e.DataDir, 0700); err != nil {
 		return "", err
 	}
@@ -118,8 +161,8 @@ func CurrentETCDOptions() (InitialOptions, error) {
 	return executor.CurrentETCDOptions()
 }
 
-func ETCD(args ETCDConfig) error {
-	return executor.ETCD(args)
+func ETCD(args ETCDConfig, extraArgs []string) error {
+	return executor.ETCD(args, extraArgs)
 }
 
 func CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error {

diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go
@@ -237,7 +237,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 				return err
 			}
 			logrus.Infof("Retrieving etcd snapshot %s from S3", e.config.ClusterResetRestorePath)
-			if err := e.s3.download(ctx); err != nil {
+			if err := e.s3.Download(ctx); err != nil {
 				return err
 			}
 			logrus.Infof("S3 download complete for %s", e.config.ClusterResetRestorePath)
@@ -605,7 +605,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
 		HeartbeatInterval: 500,
 		Logger:            "zap",
 		LogOutputs:        []string{"stderr"},
-	})
+	}, e.config.ExtraEtcdArgs)
 }
 
 // RemovePeer removes a peer from the cluster. The peer name and IP address must both match.
@@ -1024,7 +1024,7 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]Snapsho
 // if it hasn't yet been initialized.
 func (e *ETCD) initS3IfNil(ctx context.Context) error {
 	if e.s3 == nil {
-		s3, err := newS3(ctx, e.config)
+		s3, err := NewS3(ctx, e.config)
 		if err != nil {
 			return err
 		}

diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go
@@ -31,7 +31,7 @@ type S3 struct {
 // newS3 creates a new value of type s3 pointer with a
 // copy of the config.Control pointer and initializes
 // a new Minio client.
-func newS3(ctx context.Context, config *config.Control) (*S3, error) {
+func NewS3(ctx context.Context, config *config.Control) (*S3, error) {
 	tr := http.DefaultTransport
 
 	switch {
@@ -112,7 +112,7 @@ func (s *S3) upload(ctx context.Context, snapshot string) error {
 
 // download downloads the given snapshot from the configured S3
 // compatible backend.
-func (s *S3) download(ctx context.Context) error {
+func (s *S3) Download(ctx context.Context) error {
 	var remotePath string
 	if s.config.EtcdS3Folder != "" {
 		remotePath = filepath.Join(s.config.EtcdS3Folder, s.config.ClusterResetRestorePath)