From a8fedc86e10790fa034c94b372a9d0d7c3f6dd65 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Oct 2017 13:07:48 -0700 Subject: [PATCH 1/4] Create main file for storage provisioner --- Makefile | 10 ++++++++++ cmd/storage-provisioner/main.go | 19 +++++++++++++++++++ deploy/storage-provisioner/Dockerfile | 20 ++++++++++++++++++++ pkg/localkube/storage_provisioner.go | 7 +++---- 4 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 cmd/storage-provisioner/main.go create mode 100644 deploy/storage-provisioner/Dockerfile diff --git a/Makefile b/Makefile index dab6533fbd51..ef76a4ab058a 100755 --- a/Makefile +++ b/Makefile @@ -300,6 +300,16 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile @echo "" @echo "$(@) successfully built" +out/storage-provisioner-main: cmd/storage-provisioner/main.go + go build cmd/storage-provisioner/main.go + +.PHONY: storage-provisioner-image +storage-provisioner-image: out/storage-provisioner-main + docker build -t $(REGISTRY)/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile . + gcloud docker -- push $(REGISTRY)/storage-provisioner:$(TAG) + # docker build -t gcr.io/priya-wadhwa/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile . + # gcloud docker -- push gcr.io/priya-wadhwa/storage-provisioner:$(TAG) + .PHONY: release-iso release-iso: minikube_iso checksum gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso diff --git a/cmd/storage-provisioner/main.go b/cmd/storage-provisioner/main.go new file mode 100644 index 000000000000..9ee043583bee --- /dev/null +++ b/cmd/storage-provisioner/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "k8s.io/minikube/cmd/localkube/cmd" + "sync" +) + +func main() { + localkubeServer := cmd.NewLocalkubeServer() + storageProvisionerServer := localkubeServer.NewStorageProvisionerServer() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + storageProvisionerServer.Start() + }() + wg.Wait() +} diff --git a/deploy/storage-provisioner/Dockerfile b/deploy/storage-provisioner/Dockerfile new file mode 100644 index 000000000000..e14d211ea01c --- /dev/null +++ b/deploy/storage-provisioner/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:16.04 + +COPY main main + +CMD ["/main"] + diff --git a/pkg/localkube/storage_provisioner.go b/pkg/localkube/storage_provisioner.go index 46d3af277595..5d22f81845fd 100644 --- a/pkg/localkube/storage_provisioner.go +++ b/pkg/localkube/storage_provisioner.go @@ -30,8 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/minikube/pkg/util" + restclient "k8s.io/client-go/rest" ) const provisionerName = "k8s.io/minikube-hostpath" @@ -115,9 +114,8 @@ func (lk LocalkubeServer) NewStorageProvisionerServer() Server { } func StartStorageProvisioner(lk LocalkubeServer) func() error { - return func() error { - config, err := clientcmd.BuildConfigFromFlags("", util.DefaultKubeConfigPath) + config, err := restclient.InClusterConfig() if err != nil { return err } @@ -141,6 +139,7 @@ func StartStorageProvisioner(lk LocalkubeServer) func() error { // PVs pc := controller.NewProvisionController(clientset, provisionerName, hostPathProvisioner, serverVersion.GitVersion) + fmt.Println("wait never stop") pc.Run(wait.NeverStop) return nil } From 71b38d1e4e6f4fc9a0f5e09b9e50d6c8d540d03d Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Oct 2017 13:45:31 -0700 Subject: [PATCH 2/4] Tests passing with storage-provisioner in addons --- Makefile | 6 ++-- cmd/localkube/cmd/start.go | 2 -- .../storage-provisioner.yaml | 29 +++++++++++++++++++ pkg/minikube/assets/addons.go | 7 +++++ test/integration/pv_test.go | 22 ++++++++++++++ 5 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 deploy/addons/storage-provisioner/storage-provisioner.yaml diff --git a/Makefile b/Makefile index ef76a4ab058a..c68d9f0a33b5 100755 --- a/Makefile +++ b/Makefile @@ -300,15 +300,13 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile @echo "" @echo "$(@) successfully built" -out/storage-provisioner-main: cmd/storage-provisioner/main.go +storage-provisioner-main: cmd/storage-provisioner/main.go go build cmd/storage-provisioner/main.go .PHONY: storage-provisioner-image -storage-provisioner-image: out/storage-provisioner-main +storage-provisioner-image: storage-provisioner-main docker build -t $(REGISTRY)/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile . gcloud docker -- push $(REGISTRY)/storage-provisioner:$(TAG) - # docker build -t gcr.io/priya-wadhwa/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile . - # gcloud docker -- push gcr.io/priya-wadhwa/storage-provisioner:$(TAG) .PHONY: release-iso release-iso: minikube_iso checksum diff --git a/cmd/localkube/cmd/start.go b/cmd/localkube/cmd/start.go index 88c2d50d9abb..9669d9a769c9 100644 --- a/cmd/localkube/cmd/start.go +++ b/cmd/localkube/cmd/start.go @@ -139,6 +139,4 @@ func SetupServer(s *localkube.LocalkubeServer) { proxy := s.NewProxyServer() s.AddServer(proxy) - storageProvisioner := s.NewStorageProvisionerServer() - s.AddServer(storageProvisioner) } diff --git a/deploy/addons/storage-provisioner/storage-provisioner.yaml b/deploy/addons/storage-provisioner/storage-provisioner.yaml new file mode 100644 index 000000000000..71e7baa7e18e --- /dev/null +++ b/deploy/addons/storage-provisioner/storage-provisioner.yaml @@ -0,0 +1,29 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: storage-provisioner + namespace: kube-system + labels: + integration-test: storage-provisioner + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/minikube-addons: storage-provisioner +spec: + hostNetwork: true + containers: + - name: storage-provisioner + image: gcr.io/k8s-minikube/storage-provisioner:v1.8.0 + imagePullPolicy: IfNotPresent diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 43d5d6d895ae..43a88bc9c12b 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -82,6 +82,13 @@ var Addons = map[string]*Addon{ "storageclass.yaml", "0640"), }, true, "default-storageclass"), + "storage-provisioner": NewAddon([]*BinDataAsset{ + NewBinDataAsset( + "deploy/addons/storage-provisioner/storage-provisioner.yaml", + constants.AddonsPath, + "storage-provisioner.yaml", + "0640"), + }, true, "storage-provisioner"), "coredns": NewAddon([]*BinDataAsset{ NewBinDataAsset( "deploy/addons/coredns/coreDNS-controller.yaml", diff --git a/test/integration/pv_test.go b/test/integration/pv_test.go index 29c8013d1047..ec09c46a91cf 100644 --- a/test/integration/pv_test.go +++ b/test/integration/pv_test.go @@ -20,12 +20,15 @@ package integration import ( "fmt" + "github.com/pkg/errors" "path/filepath" "testing" "time" + "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/storage" + commonutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/test/integration/util" ) @@ -58,6 +61,25 @@ func testProvisioning(t *testing.T) { t.Fatalf("No default storage class: %s", err) } + // Check that the storage provisioner pod is running + + checkPodRunning := func() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"})) + + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return err + } + return nil + } + + if err := checkPodRunning(); err != nil { + t.Fatal("Check storage-provisioner pod running failed with error: ", err) + } + // Now create the PVC pvcPath := filepath.Join(*testdataDir, "pvc.yaml") if _, err := kubectlRunner.RunCommand([]string{"create", "-f", pvcPath}); err != nil { From 7098431c6ac0f72a999424d6e41436dd2afa650f Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 30 Oct 2017 13:50:57 -0700 Subject: [PATCH 3/4] Small changes --- cmd/storage-provisioner/main.go | 16 ++++++++++++++++ .../storage-provisioner/storage-provisioner.yaml | 2 +- deploy/storage-provisioner/Dockerfile | 4 +--- pkg/minikube/constants/constants.go | 6 ++++++ test/integration/functional_test.go | 6 +----- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/cmd/storage-provisioner/main.go b/cmd/storage-provisioner/main.go index 9ee043583bee..335999319ed8 100644 --- a/cmd/storage-provisioner/main.go +++ b/cmd/storage-provisioner/main.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/deploy/addons/storage-provisioner/storage-provisioner.yaml b/deploy/addons/storage-provisioner/storage-provisioner.yaml index 71e7baa7e18e..1bef2d8eb6d9 100644 --- a/deploy/addons/storage-provisioner/storage-provisioner.yaml +++ b/deploy/addons/storage-provisioner/storage-provisioner.yaml @@ -19,7 +19,7 @@ metadata: namespace: kube-system labels: integration-test: storage-provisioner - addonmanager.kubernetes.io/mode: Reconcile + addonmanager.kubernetes.io/mode: EnsureExists kubernetes.io/minikube-addons: storage-provisioner spec: hostNetwork: true diff --git a/deploy/storage-provisioner/Dockerfile b/deploy/storage-provisioner/Dockerfile index e14d211ea01c..dc0decc3bd4c 100644 --- a/deploy/storage-provisioner/Dockerfile +++ b/deploy/storage-provisioner/Dockerfile @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:16.04 - +FROM scratch COPY main main - CMD ["/main"] diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 2c687c773a95..2a047368780c 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -181,6 +181,9 @@ var LocalkubeCachedImages = []string{ // Pause "gcr.io/google_containers/pause-amd64:3.0", + + //Storage Provisioner + "gcr.io/k8s-minikube/storage-provisioner:v1.8.0", } func GetKubeadmCachedImages(version string) []string { @@ -206,6 +209,9 @@ func GetKubeadmCachedImages(version string) []string { "gcr.io/google_containers/kube-scheduler-amd64:" + version, "gcr.io/google_containers/kube-controller-manager-amd64:" + version, "gcr.io/google_containers/kube-apiserver-amd64:" + version, + + //Storage Provisioner + "gcr.io/k8s-minikube/storage-provisioner:v1.8.0", } } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 93233b76583f..f9f16b1d5029 100755 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -35,11 +35,7 @@ func TestFunctional(t *testing.T) { t.Run("Addons", testAddons) t.Run("Dashboard", testDashboard) t.Run("ServicesList", testServicesList) - - // Don't run this test on kubeadm bootstrapper for now. - if !strings.Contains(*args, "--bootstrapper=kubeadm") { - t.Run("Provisioning", testProvisioning) - } + t.Run("Provisioning", testProvisioning) if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") { t.Run("EnvVars", testClusterEnv) From c342ed4d5721342068ec0b5cb0a016aaea960df3 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 1 Nov 2017 10:47:08 -0700 Subject: [PATCH 4/4] Added storage prov deps to Makefile and rewrote main file --- Makefile | 8 +-- cmd/storage-provisioner/main.go | 21 +++---- .../storage-provisioner.yaml | 1 - deploy/storage-provisioner/Dockerfile | 5 +- pkg/localkube/storage_provisioner.go | 59 +++++++++---------- 5 files changed, 42 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index c68d9f0a33b5..655dc39b9411 100755 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ LOCALKUBE_LDFLAGS := "$(K8S_VERSION_LDFLAGS) $(MINIKUBE_LDFLAGS) -s -w -extldfla LOCALKUBEFILES := GOPATH=$(GOPATH) go list -f '{{join .Deps "\n"}}' ./cmd/localkube/ | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}' MINIKUBEFILES := GOPATH=$(GOPATH) go list -f '{{join .Deps "\n"}}' ./cmd/minikube/ | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}' HYPERKIT_FILES := GOPATH=$(GOPATH) go list -f '{{join .Deps "\n"}}' k8s.io/minikube/cmd/drivers/hyperkit | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}' - +STORAGE_PROVISIONER_FILES := GOPATH=$(GOPATH) go list -f '{{join .Deps "\n"}}' k8s.io/minikube/cmd/storage-provisioner | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}' MINIKUBE_TEST_FILES := go list -f '{{ if .TestGoFiles }} {{.ImportPath}} {{end}}' ./... | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}' KVM_DRIVER_FILES := $(shell go list -f '{{join .Deps "\n"}}' ./cmd/drivers/kvm/ | grep k8s.io | xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}') @@ -300,11 +300,11 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile @echo "" @echo "$(@) successfully built" -storage-provisioner-main: cmd/storage-provisioner/main.go - go build cmd/storage-provisioner/main.go +out/storage-provisioner: $(shell $(STORAGE_PROVISIONER_FILES)) + go build -o $(BUILD_DIR)/storage-provisioner cmd/storage-provisioner/main.go .PHONY: storage-provisioner-image -storage-provisioner-image: storage-provisioner-main +storage-provisioner-image: out/storage-provisioner docker build -t $(REGISTRY)/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile . gcloud docker -- push $(REGISTRY)/storage-provisioner:$(TAG) diff --git a/cmd/storage-provisioner/main.go b/cmd/storage-provisioner/main.go index 335999319ed8..ac08d4ae73d0 100644 --- a/cmd/storage-provisioner/main.go +++ b/cmd/storage-provisioner/main.go @@ -17,19 +17,16 @@ limitations under the License. package main import ( - "k8s.io/minikube/cmd/localkube/cmd" - "sync" + "flag" + "github.com/golang/glog" + "k8s.io/minikube/pkg/localkube" ) func main() { - localkubeServer := cmd.NewLocalkubeServer() - storageProvisionerServer := localkubeServer.NewStorageProvisionerServer() - - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - storageProvisionerServer.Start() - }() - wg.Wait() + flag.Parse() + + if err := localkube.StartStorageProvisioner(); err != nil { + glog.Exit(err) + } + } diff --git a/deploy/addons/storage-provisioner/storage-provisioner.yaml b/deploy/addons/storage-provisioner/storage-provisioner.yaml index 1bef2d8eb6d9..2abdae67aea0 100644 --- a/deploy/addons/storage-provisioner/storage-provisioner.yaml +++ b/deploy/addons/storage-provisioner/storage-provisioner.yaml @@ -20,7 +20,6 @@ metadata: labels: integration-test: storage-provisioner addonmanager.kubernetes.io/mode: EnsureExists - kubernetes.io/minikube-addons: storage-provisioner spec: hostNetwork: true containers: diff --git a/deploy/storage-provisioner/Dockerfile b/deploy/storage-provisioner/Dockerfile index dc0decc3bd4c..bef335d5142c 100644 --- a/deploy/storage-provisioner/Dockerfile +++ b/deploy/storage-provisioner/Dockerfile @@ -13,6 +13,5 @@ # limitations under the License. FROM scratch -COPY main main -CMD ["/main"] - +COPY out/storage-provisioner storage-provisioner +CMD ["/storage-provisioner"] diff --git a/pkg/localkube/storage_provisioner.go b/pkg/localkube/storage_provisioner.go index 5d22f81845fd..a2baba170b74 100644 --- a/pkg/localkube/storage_provisioner.go +++ b/pkg/localkube/storage_provisioner.go @@ -109,38 +109,33 @@ func (p *hostPathProvisioner) Delete(volume *v1.PersistentVolume) error { return nil } -func (lk LocalkubeServer) NewStorageProvisionerServer() Server { - return NewSimpleServer("storage-provisioner", serverInterval, StartStorageProvisioner(lk), noop) -} +// Start storage provisioner server +func StartStorageProvisioner() error { + config, err := restclient.InClusterConfig() + if err != nil { + return err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + glog.Fatalf("Failed to create client: %v", err) + } -func StartStorageProvisioner(lk LocalkubeServer) func() error { - return func() error { - config, err := restclient.InClusterConfig() - if err != nil { - return err - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - glog.Fatalf("Failed to create client: %v", err) - } - - // The controller needs to know what the server version is because out-of-tree - // provisioners aren't officially supported until 1.5 - serverVersion, err := clientset.Discovery().ServerVersion() - if err != nil { - return fmt.Errorf("Error getting server version: %v", err) - } - - // Create the provisioner: it implements the Provisioner interface expected by - // the controller - hostPathProvisioner := NewHostPathProvisioner() - - // Start the provision controller which will dynamically provision hostPath - // PVs - pc := controller.NewProvisionController(clientset, provisionerName, hostPathProvisioner, serverVersion.GitVersion) - - fmt.Println("wait never stop") - pc.Run(wait.NeverStop) - return nil + // The controller needs to know what the server version is because out-of-tree + // provisioners aren't officially supported until 1.5 + serverVersion, err := clientset.Discovery().ServerVersion() + if err != nil { + return fmt.Errorf("Error getting server version: %v", err) } + + // Create the provisioner: it implements the Provisioner interface expected by + // the controller + hostPathProvisioner := NewHostPathProvisioner() + + // Start the provision controller which will dynamically provision hostPath + // PVs + pc := controller.NewProvisionController(clientset, provisionerName, hostPathProvisioner, serverVersion.GitVersion) + + glog.Info("Starting storage provisioner server") + pc.Run(wait.NeverStop) + return nil }