From 46d61331cd327fca05e269510ec4f7a88a70b234 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 2 Aug 2019 22:51:59 -0700 Subject: [PATCH 1/5] separate certs per profile --- cmd/minikube/cmd/start.go | 11 ++++++----- go.mod | 1 + go.sum | 3 +++ pkg/minikube/bootstrapper/bootstrapper.go | 2 +- pkg/minikube/bootstrapper/certs.go | 12 ++++++------ pkg/minikube/bootstrapper/certs_test.go | 7 ++++--- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 ++-- 7 files changed, 23 insertions(+), 17 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index b697453a7dab..969c517a1547 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -807,7 +807,7 @@ func setupKubeAdm(mAPI libmachine.API, kc cfg.KubernetesConfig) bootstrapper.Boo if err := bs.UpdateCluster(kc); err != nil { exit.WithError("Failed to update cluster", err) } - if err := bs.SetupCerts(kc); err != nil { + if err := bs.SetupCerts(kc, cfg.GetMachineName()); err != nil { exit.WithError("Failed to setup certs", err) } return bs @@ -824,13 +824,14 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *pkgutil.KubeConfigSetup { if c.KubernetesConfig.APIServerName != constants.APIServerName { addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1) } + p := cfg.GetMachineName() kcs := &pkgutil.KubeConfigSetup{ - ClusterName: cfg.GetMachineName(), + ClusterName: p, ClusterServerAddress: addr, - ClientCertificate: constants.MakeMiniPath("client.crt"), - ClientKey: constants.MakeMiniPath("client.key"), - CertificateAuthority: constants.MakeMiniPath("ca.crt"), + ClientCertificate: filepath.Join(constants.GetProfilePath(p), "client.crt"), + ClientKey: filepath.Join(constants.GetProfilePath(p), "client.key"), + CertificateAuthority: filepath.Join(constants.GetProfilePath(p), "ca.crt"), KeepContext: viper.GetBool(keepContext), EmbedCerts: viper.GetBool(embedCerts), } diff --git a/go.mod b/go.mod index 955c41f6edcc..04a7887c65a7 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/sirupsen/logrus v1.4.1 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index 670d0201fde1..f423eb762ca3 100644 --- a/go.sum +++ b/go.sum @@ -194,6 +194,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -398,6 +399,8 @@ github.com/seccomp/libseccomp-golang v0.0.0-20150813023252-1b506fc7c24e/go.mod h github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index b3ff31095d76..2450cfd847d8 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -42,7 +42,7 @@ type Bootstrapper interface { WaitCluster(config.KubernetesConfig) error // LogCommands returns a map of log type to a command which will display that log. LogCommands(LogOptions) map[string]string - SetupCerts(cfg config.KubernetesConfig) error + SetupCerts(cfg config.KubernetesConfig, profile string) error GetKubeletStatus() (string, error) GetAPIServerStatus(net.IP, int) (string, error) } diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index b24f2e7d830b..36648e9bb4a2 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -43,11 +43,11 @@ var ( ) // SetupCerts gets the generated credentials required to talk to the APIServer. -func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { - localPath := constants.GetMinipath() - glog.Infof("Setting up certificates for IP: %s\n", k8s.NodeIP) +func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, profile string) error { + localPath := constants.GetProfilePath(profile) + glog.Infof("Setting up certificates for profile %s IP: %s\n", profile, k8s.NodeIP) - if err := generateCerts(k8s); err != nil { + if err := generateCerts(k8s, profile); err != nil { return errors.Wrap(err, "Error generating certs") } @@ -97,13 +97,13 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error { return nil } -func generateCerts(k8s config.KubernetesConfig) error { +func generateCerts(k8s config.KubernetesConfig, profile string) error { serviceIP, err := util.GetServiceClusterIP(k8s.ServiceCIDR) if err != nil { return errors.Wrap(err, "getting service cluster ip") } - localPath := constants.GetMinipath() + localPath := constants.GetProfilePath(profile) caCertPath := filepath.Join(localPath, "ca.crt") caKeyPath := filepath.Join(localPath, "ca.key") diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 893c7a083240..3bd74cf1ce7f 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -29,7 +29,8 @@ import ( ) func TestSetupCerts(t *testing.T) { - tempDir := tests.MakeTempDir() + p := t.Name() + tempDir := tests.MakeTempDir() // TODO: this doesn't seem to be used defer os.RemoveAll(tempDir) f := command.NewFakeCommandRunner() @@ -41,10 +42,10 @@ func TestSetupCerts(t *testing.T) { var filesToBeTransferred []string for _, cert := range certs { - filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetMinipath(), cert)) + filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetProfilePath(p), cert)) } - if err := SetupCerts(f, k8s); err != nil { + if err := SetupCerts(f, k8s, p); err != nil { t.Fatalf("Error starting cluster: %v", err) } for _, cert := range filesToBeTransferred { diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 9e45a9f9632d..96e3d87b9e02 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -422,8 +422,8 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { } // SetupCerts sets up certificates within the cluster. -func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error { - return bootstrapper.SetupCerts(k.c, k8s) +func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig, profile string) error { + return bootstrapper.SetupCerts(k.c, k8s, profile) } // NewKubeletConfig generates a new systemd unit containing a configured kubelet From 5af348c4109ae61d86ce57c990ed7ee71ef57531 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sat, 3 Aug 2019 00:03:03 -0700 Subject: [PATCH 2/5] add logging --- go.mod | 1 - go.sum | 3 --- pkg/util/crypto.go | 7 +++++++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 04a7887c65a7..955c41f6edcc 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect github.com/shirou/gopsutil v2.18.12+incompatible - github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/sirupsen/logrus v1.4.1 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index f423eb762ca3..670d0201fde1 100644 --- a/go.sum +++ b/go.sum @@ -194,7 +194,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -399,8 +398,6 @@ github.com/seccomp/libseccomp-golang v0.0.0-20150813023252-1b506fc7c24e/go.mod h github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= diff --git a/pkg/util/crypto.go b/pkg/util/crypto.go index dc8c552cd3e1..6b18dc3b8723 100644 --- a/pkg/util/crypto.go +++ b/pkg/util/crypto.go @@ -23,6 +23,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "io/ioutil" "math/big" "net" @@ -35,6 +36,8 @@ import ( // GenerateCACert generates a CA certificate and RSA key for a common name func GenerateCACert(certPath, keyPath string, name string) error { + fmt.Printf("Inside GenerateCACert certpath %s keypath %s", certPath, keyPath) + priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return errors.Wrap(err, "Error generating rsa key") @@ -64,6 +67,8 @@ func GenerateCACert(certPath, keyPath string, name string) error { // GenerateSignedCert generates a signed certificate and key func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS []string, signerCertPath, signerKeyPath string) error { + fmt.Printf("Inside GenerateSignedCert certpath %s keypath %s", certPath, keyPath) + fmt.Println() signerCertBytes, err := ioutil.ReadFile(signerCertPath) if err != nil { return errors.Wrap(err, "Error reading file: signerCertPath") @@ -115,6 +120,7 @@ func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS } func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) { + fmt.Printf("Inside loadOrGeneratePrivateKey keypath %s", keyPath) keyBytes, err := ioutil.ReadFile(keyPath) if err == nil { decodedKey, _ := pem.Decode(keyBytes) @@ -133,6 +139,7 @@ func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) { } func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *rsa.PrivateKey, keyPath string, parent *x509.Certificate, signingKey *rsa.PrivateKey) error { + fmt.Printf("Inside writeCertsAndKeys : certPath %s keypath %s", certPath, keyPath) derBytes, err := x509.CreateCertificate(rand.Reader, template, parent, &signeeKey.PublicKey, signingKey) if err != nil { return errors.Wrap(err, "Error creating certificate") From 234c8b3d7455cff6e1f6daecd62cc53653b58dc9 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sat, 3 Aug 2019 01:39:13 -0700 Subject: [PATCH 3/5] fix ca certs after stop for paralell --- cmd/minikube/cmd/root.go | 1 + cmd/minikube/cmd/start.go | 6 +++--- pkg/minikube/bootstrapper/certs.go | 2 +- pkg/minikube/bootstrapper/certs_test.go | 2 +- pkg/util/crypto.go | 7 ------- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 07c42b58fd8f..4cb95450e85f 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -42,6 +42,7 @@ import ( var dirs = [...]string{ constants.GetMinipath(), + constants.MakeMiniPath("profiles", config.GetMachineName(), "certs"), constants.MakeMiniPath("certs"), constants.MakeMiniPath("machines"), constants.MakeMiniPath("cache"), diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 969c517a1547..72280de22561 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -829,9 +829,9 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *pkgutil.KubeConfigSetup { kcs := &pkgutil.KubeConfigSetup{ ClusterName: p, ClusterServerAddress: addr, - ClientCertificate: filepath.Join(constants.GetProfilePath(p), "client.crt"), - ClientKey: filepath.Join(constants.GetProfilePath(p), "client.key"), - CertificateAuthority: filepath.Join(constants.GetProfilePath(p), "ca.crt"), + ClientCertificate: filepath.Join(constants.GetProfilePath(p), "certs", "client.crt"), + ClientKey: filepath.Join(constants.GetProfilePath(p), "certs", "client.key"), + CertificateAuthority: filepath.Join(constants.GetProfilePath(p), "certs", "ca.crt"), KeepContext: viper.GetBool(keepContext), EmbedCerts: viper.GetBool(embedCerts), } diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 36648e9bb4a2..d21c75038c9b 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -103,7 +103,7 @@ func generateCerts(k8s config.KubernetesConfig, profile string) error { return errors.Wrap(err, "getting service cluster ip") } - localPath := constants.GetProfilePath(profile) + localPath := filepath.Join(constants.GetProfilePath(profile), "certs") caCertPath := filepath.Join(localPath, "ca.crt") caKeyPath := filepath.Join(localPath, "ca.key") diff --git a/pkg/minikube/bootstrapper/certs_test.go b/pkg/minikube/bootstrapper/certs_test.go index 3bd74cf1ce7f..188db0db4a80 100644 --- a/pkg/minikube/bootstrapper/certs_test.go +++ b/pkg/minikube/bootstrapper/certs_test.go @@ -42,7 +42,7 @@ func TestSetupCerts(t *testing.T) { var filesToBeTransferred []string for _, cert := range certs { - filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetProfilePath(p), cert)) + filesToBeTransferred = append(filesToBeTransferred, filepath.Join(constants.GetProfilePath(p), "certs", cert)) } if err := SetupCerts(f, k8s, p); err != nil { diff --git a/pkg/util/crypto.go b/pkg/util/crypto.go index 6b18dc3b8723..dc8c552cd3e1 100644 --- a/pkg/util/crypto.go +++ b/pkg/util/crypto.go @@ -23,7 +23,6 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" - "fmt" "io/ioutil" "math/big" "net" @@ -36,8 +35,6 @@ import ( // GenerateCACert generates a CA certificate and RSA key for a common name func GenerateCACert(certPath, keyPath string, name string) error { - fmt.Printf("Inside GenerateCACert certpath %s keypath %s", certPath, keyPath) - priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return errors.Wrap(err, "Error generating rsa key") @@ -67,8 +64,6 @@ func GenerateCACert(certPath, keyPath string, name string) error { // GenerateSignedCert generates a signed certificate and key func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS []string, signerCertPath, signerKeyPath string) error { - fmt.Printf("Inside GenerateSignedCert certpath %s keypath %s", certPath, keyPath) - fmt.Println() signerCertBytes, err := ioutil.ReadFile(signerCertPath) if err != nil { return errors.Wrap(err, "Error reading file: signerCertPath") @@ -120,7 +115,6 @@ func GenerateSignedCert(certPath, keyPath, cn string, ips []net.IP, alternateDNS } func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) { - fmt.Printf("Inside loadOrGeneratePrivateKey keypath %s", keyPath) keyBytes, err := ioutil.ReadFile(keyPath) if err == nil { decodedKey, _ := pem.Decode(keyBytes) @@ -139,7 +133,6 @@ func loadOrGeneratePrivateKey(keyPath string) (*rsa.PrivateKey, error) { } func writeCertsAndKeys(template *x509.Certificate, certPath string, signeeKey *rsa.PrivateKey, keyPath string, parent *x509.Certificate, signingKey *rsa.PrivateKey) error { - fmt.Printf("Inside writeCertsAndKeys : certPath %s keypath %s", certPath, keyPath) derBytes, err := x509.CreateCertificate(rand.Reader, template, parent, &signeeKey.PublicKey, signingKey) if err != nil { return errors.Wrap(err, "Error creating certificate") From 6a201fda63e0fcc6965a7fe36276953f60e7946f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sat, 3 Aug 2019 01:49:08 -0700 Subject: [PATCH 4/5] fix cert path --- pkg/minikube/bootstrapper/certs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index d21c75038c9b..ede110eec15d 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -44,7 +44,7 @@ var ( // SetupCerts gets the generated credentials required to talk to the APIServer. func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, profile string) error { - localPath := constants.GetProfilePath(profile) + localPath := filepath.Join(constants.GetProfilePath(profile), "certs") glog.Infof("Setting up certificates for profile %s IP: %s\n", profile, k8s.NodeIP) if err := generateCerts(k8s, profile); err != nil { From ae03b3148bb2f8c0db51f92e13457bfa29ef8088 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Sat, 3 Aug 2019 16:50:33 -0700 Subject: [PATCH 5/5] stop kubelet and start cluster --- cmd/minikube/cmd/start.go | 6 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 76 +++++++++++++++----- pkg/util/constants.go | 10 +-- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 72280de22561..dc4ae55db67c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -803,13 +803,13 @@ func setupKubeAdm(mAPI libmachine.API, kc cfg.KubernetesConfig) bootstrapper.Boo for _, eo := range extraOptions { out.T(out.Option, "{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value}) } + if err := bs.SetupCerts(kc, cfg.GetMachineName()); err != nil { + exit.WithError("Failed to setup certs", err) + } // Loads cached images, generates config files, download binaries if err := bs.UpdateCluster(kc); err != nil { exit.WithError("Failed to update cluster", err) } - if err := bs.SetupCerts(kc, cfg.GetMachineName()); err != nil { - exit.WithError("Failed to setup certs", err) - } return bs } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 96e3d87b9e02..17793ebfcb13 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -38,6 +38,7 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" + cfg "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/machine" @@ -197,6 +198,37 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "parsing kubernetes version") } + err = k.kubeadmInit(k8s, false) + if err != nil { + return errors.Wrap(err, "kubeadm init phase") + } + + if version.LT(semver.MustParse("1.10.0-alpha.0")) { + // TODO(r2d4): get rid of global here + master = k8s.NodeName + if err := util.RetryAfter(200, unmarkMaster, time.Second*1); err != nil { + return errors.Wrap(err, "timed out waiting to unmark master") + } + } + + glog.Infof("Configuring cluster permissions ...") + if err := util.RetryAfter(100, elevateKubeSystemPrivileges, time.Millisecond*500); err != nil { + return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") + } + + if err := k.adjustResourceLimits(); err != nil { + glog.Warningf("unable to adjust resource limits: %v", err) + } + return nil +} + +// sorry for the stutter ! +// kubeadmInit runs kubeadm init +func (k *Bootstrapper) kubeadmInit(k8s config.KubernetesConfig, ignoreAll bool) error { + version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return errors.Wrap(err, "parsing k8s version in kubeadminit") + } extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions) @@ -217,6 +249,9 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { } ignore = append(ignore, SkipAdditionalPreflights[r.Name()]...) + if ignoreAll { + ignore = []string{"all"} + } // Allow older kubeadm versions to function with newer Docker releases. if version.LT(semver.MustParse("1.13.0")) { glog.Infof("Older Kubernetes release detected (%s), disabling SystemVerification check.", version) @@ -229,23 +264,6 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { if err != nil { return errors.Wrapf(err, "cmd failed: %s\n%s\n", cmd, out) } - - if version.LT(semver.MustParse("1.10.0-alpha.0")) { - // TODO(r2d4): get rid of global here - master = k8s.NodeName - if err := util.RetryAfter(200, unmarkMaster, time.Second*1); err != nil { - return errors.Wrap(err, "timed out waiting to unmark master") - } - } - - glog.Infof("Configuring cluster permissions ...") - if err := util.RetryAfter(100, elevateKubeSystemPrivileges, time.Millisecond*500); err != nil { - return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges") - } - - if err := k.adjustResourceLimits(); err != nil { - glog.Warningf("unable to adjust resource limits: %v", err) - } return nil } @@ -359,6 +377,30 @@ func (k *Bootstrapper) RestartCluster(k8s config.KubernetesConfig) error { // Run commands one at a time so that it is easier to root cause failures. for _, cmd := range cmds { if err := k.c.Run(cmd); err != nil { + // handle cert errors on restart ! + // context https://github.com/kubernetes/minikube/pull/4968#issuecomment-517953773 + + if strings.Contains(err.Error(), "cert") { + glog.Errorf("found wrong certs on the VM, will reconfigure") + err = k.c.Run(`sudo systemctl stop kubelet`) + if err != nil { + glog.Errorf("error stopping kubelet for fixing cert issue %v", err) + } + err = k.SetupCerts(k8s, cfg.GetMachineName()) + if err != nil { + glog.Errorf("error setting up certs %v", err) + } + err = k.kubeadmInit(k8s, true) + if err != nil { + glog.Errorf("error kubeadm init %v", err) + } + err = k.StartCluster(k8s) + if err != nil { + glog.Errorf("error starting the cluster %v", err) + } + + continue + } return errors.Wrapf(err, "running cmd: %s", cmd) } } diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 92f0c29bf5fb..90a51f25fe41 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -26,10 +26,12 @@ import ( const ( APIServerPort = 8443 DefaultMinikubeDirectory = "/var/lib/minikube" - DefaultCertPath = DefaultMinikubeDirectory + "/certs/" - DefaultKubeConfigPath = DefaultMinikubeDirectory + "/kubeconfig" - DefaultDNSDomain = "cluster.local" - DefaultServiceCIDR = "10.96.0.0/12" + // changing certs folder to profilecerts. since minikube 1.3 we generate certs per profile, to keep the version upgrade without conflict + // https://github.com/kubernetes/minikube/pull/4968 + DefaultCertPath = DefaultMinikubeDirectory + "/profilecerts/" + DefaultKubeConfigPath = DefaultMinikubeDirectory + "/kubeconfig" + DefaultDNSDomain = "cluster.local" + DefaultServiceCIDR = "10.96.0.0/12" ) // DefaultV114AdmissionControllers are admission controllers we default to in v1.14.x