diff --git a/go.mod b/go.mod index 00cad77311b9..faec61e4f512 100644 --- a/go.mod +++ b/go.mod @@ -123,7 +123,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/json-iterator/go v1.1.12 github.com/k3s-io/helm-controller v0.15.9 - github.com/k3s-io/kine v0.11.4 + github.com/k3s-io/kine v0.11.5 github.com/klauspost/compress v1.17.2 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.10.2 diff --git a/go.sum b/go.sum index 8e5c0e40d570..07e801121324 100644 --- a/go.sum +++ b/go.sum @@ -959,8 +959,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64= github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= -github.com/k3s-io/kine v0.11.4 h1:ZIXQT4vPPKNL9DwLF4dQ11tWtpJ1C/7OKNIpFmTkImo= -github.com/k3s-io/kine v0.11.4/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4= +github.com/k3s-io/kine v0.11.5 h1:/oqepXJblSL3sYLhYDa22mRhrA3CVO6DzKcipnFdO8U= +github.com/k3s-io/kine v0.11.5/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4= github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0= github.com/k3s-io/klog/v2 v2.100.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= github.com/k3s-io/kube-router/v2 v2.0.1 h1:UCsdkQjSfOkVakixilRDDkG9yq775GBSKxBfsyUj8ng= diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index dd460a9602c4..e746549992be 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -60,6 +60,7 @@ type Server struct { DatastoreCAFile string DatastoreCertFile string DatastoreKeyFile string + KineTLS bool AdvertiseIP string AdvertisePort int DisableScheduler bool @@ -310,6 +311,11 @@ var ServerFlags = []cli.Flag{ Usage: "(flags) Customized flag for kube-cloud-controller-manager process", Value: &ServerConfig.ExtraCloudControllerArgs, }, + &cli.BoolFlag{ + Name: "kine-tls", + Usage: "(db) Enable tls for kine", + Destination: &ServerConfig.KineTLS, + }, &cli.StringFlag{ Name: "datastore-endpoint", Usage: "(db) Specify etcd, NATS, MySQL, Postgres, or SQLite (default) data source name", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 1fc78d6ab0d9..f322695b1291 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -49,9 +49,7 @@ func RunWithControllers(app *cli.Context, leaderControllers server.CustomControl } func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomControllers, controllers server.CustomControllers) error { - var ( - err error - ) + var err error // Validate build env cmds.MustValidateGolang() @@ -154,6 +152,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.Datastore.BackendTLSConfig.CAFile = cfg.DatastoreCAFile serverConfig.ControlConfig.Datastore.BackendTLSConfig.CertFile = cfg.DatastoreCertFile serverConfig.ControlConfig.Datastore.BackendTLSConfig.KeyFile = cfg.DatastoreKeyFile + serverConfig.ControlConfig.KineTLS = cfg.KineTLS serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index d87c2d9fa617..7e9badc830b1 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -84,7 +84,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { return nil, err } - if err := c.startStorage(ctx); err != nil { + if err := c.startStorage(ctx, false); err != nil { return nil, err } @@ -132,12 +132,19 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { // This calls into the kine endpoint code, which sets up the database client // and unix domain socket listener if using an external database. In the case of an etcd // backend it just returns the user-provided etcd endpoints and tls config. -func (c *Cluster) startStorage(ctx context.Context) error { - if c.storageStarted { +func (c *Cluster) startStorage(ctx context.Context, bootstrap bool) error { + if c.storageStarted && !c.config.KineTLS { return nil } c.storageStarted = true + if !bootstrap { + // set the tls config for the kine storage + c.config.Datastore.ServerTLSConfig.CAFile = c.config.Runtime.ETCDServerCA + c.config.Datastore.ServerTLSConfig.CertFile = c.config.Runtime.ServerETCDCert + c.config.Datastore.ServerTLSConfig.KeyFile = c.config.Runtime.ServerETCDKey + } + // start listening on the kine socket as an etcd endpoint, or return the external etcd endpoints etcdConfig, err := endpoint.Listen(ctx, c.config.Datastore) if err != nil { @@ -148,9 +155,15 @@ func (c *Cluster) startStorage(ctx context.Context) error { // based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require // leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers. c.config.Runtime.EtcdConfig = etcdConfig - c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig - c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",") - c.config.NoLeaderElect = !etcdConfig.LeaderElect + + // after the bootstrap we need to set the args for api-server with kine in unixs or just set the + // values if the datastoreTLS is not enabled + if !bootstrap || !c.config.KineTLS { + c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig + c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",") + c.config.NoLeaderElect = !etcdConfig.LeaderElect + } + return nil } diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 549291961253..1d509b8db5f6 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -20,7 +20,6 @@ import ( const maxBootstrapWaitAttempts = 5 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { - token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err @@ -151,7 +150,7 @@ func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client // bootstrap key as a lock. This function will not return successfully until either the // bootstrap key has been locked, or data is read into the struct. func (c *Cluster) storageBootstrap(ctx context.Context) error { - if err := c.startStorage(ctx); err != nil { + if err := c.startStorage(ctx, true); err != nil { return err } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 4a4072648315..6abf393b1d15 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -179,6 +179,7 @@ type Control struct { KubeConfigMode string HelmJobImage string DataDir string + KineTLS bool Datastore endpoint.Config `json:"-"` Disables map[string]bool DisableAPIServer bool diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 140fb2edd0cc..fd370130c9ae 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -42,8 +42,7 @@ const ( RequestHeaderCN = "system:auth-proxy" ) -var ( - kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1 +var kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1 clusters: - cluster: server: {{.URL}} @@ -64,7 +63,6 @@ users: client-certificate: {{.ClientCert}} client-key: {{.ClientKey}} `)) -) func migratePassword(p *passwd.Passwd) error { server, _ := p.Pass("server") @@ -283,9 +281,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control) error { } func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) { - var ( - err error - ) + var err error serverPass := config.Token if serverPass == "" { @@ -448,14 +444,16 @@ func genServerCerts(config *config.Control) error { } func genETCDCerts(config *config.Control) error { - runtime := config.Runtime regen, err := createSigningCertKey("etcd-server", runtime.ETCDServerCA, runtime.ETCDServerCAKey) if err != nil { return err } - altNames := &certutil.AltNames{} + altNames := &certutil.AltNames{ + DNSNames: []string{"kine.sock"}, + } + addSANs(altNames, config.SANs) if _, err := createClientCertKey(regen, "etcd-client", nil, @@ -845,5 +843,4 @@ func genCloudConfig(controlConfig *config.Control) error { return err } return os.WriteFile(controlConfig.Runtime.CloudControllerConfig, b, 0600) - } diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 5477e8ae05d4..b99fc7cf1eb3 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -286,7 +286,6 @@ func prepare(ctx context.Context, config *config.Control) error { deps.CreateRuntimeCertFiles(config) cluster := cluster.New(config) - if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil { return err } diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go index 13f9c36a3498..53f387aa899e 100644 --- a/tests/integration/startup/startup_int_test.go +++ b/tests/integration/startup/startup_int_test.go @@ -1,6 +1,7 @@ package integration import ( + "errors" "os" "path/filepath" "testing" @@ -13,9 +14,11 @@ import ( . "github.com/onsi/gomega/gstruct" ) -var startupServer *testutil.K3sServer -var startupServerArgs = []string{} -var testLock int +var ( + startupServer *testutil.K3sServer + startupServerArgs = []string{} + testLock int +) var _ = BeforeSuite(func() { if testutil.IsExistingServer() { @@ -27,7 +30,6 @@ var _ = BeforeSuite(func() { }) var _ = Describe("startup tests", Ordered, func() { - When("a default server is created", func() { It("is created with no arguments", func() { var err error @@ -39,6 +41,59 @@ var _ = Describe("startup tests", Ordered, func() { return testutil.K3sDefaultDeployments() }, "120s", "5s").Should(Succeed()) }) + It("has kine without tls", func() { + Eventually(func() error { + match, err := testutil.SearchK3sLog(startupServer, "Kine available at unix://kine.sock") + if err != nil { + return err + } + if match { + return nil + } + return errors.New("error finding kine sock") + }, "30s", "2s").Should(Succeed()) + }) + It("does not use kine with tls after bootstrap", func() { + Eventually(func() error { + match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock") + if err != nil { + return err + } + if match { + return errors.New("Kine with tls when the kine-tls is not set") + } + return nil + }, "30s", "2s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + }) + }) + When("a server with kine-tls is created", func() { + It("is created with kine-tls", func() { + var err error + startupServerArgs = []string{"--kine-tls"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "120s", "5s").Should(Succeed()) + }) + It("set kine to use tls", func() { + Eventually(func() error { + match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock") + if err != nil { + return err + } + if match { + return nil + } + return errors.New("error finding unixs://kine.sock") + }, "30s", "2s").Should(Succeed()) + }) It("dies cleanly", func() { Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) @@ -263,13 +318,14 @@ var _ = Describe("startup tests", Ordered, func() { Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) }) }) - }) -var failed bool -var _ = AfterEach(func() { - failed = failed || CurrentSpecReport().Failed() -}) +var ( + failed bool + _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() + }) +) var _ = AfterSuite(func() { if !testutil.IsExistingServer() {