From d9fe5afa866381ba15c86930bce2bc36e3804e54 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Wed, 4 Mar 2020 23:36:37 +0800 Subject: [PATCH 1/5] Adds MetalLB addon --- .../addons/metallb/metallb-config.yaml.tmpl | 12 + deploy/addons/metallb/metallb.yaml | 293 ++++++++++++++++++ pkg/addons/config.go | 5 + pkg/minikube/assets/addons.go | 30 +- pkg/minikube/config/types.go | 28 +- 5 files changed, 349 insertions(+), 19 deletions(-) create mode 100644 deploy/addons/metallb/metallb-config.yaml.tmpl create mode 100644 deploy/addons/metallb/metallb.yaml diff --git a/deploy/addons/metallb/metallb-config.yaml.tmpl b/deploy/addons/metallb/metallb-config.yaml.tmpl new file mode 100644 index 000000000000..d89de8f582da --- /dev/null +++ b/deploy/addons/metallb/metallb-config.yaml.tmpl @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - {{ .LoadBalancerStartIP }}-{{ .LoadBalancerEndIP }} diff --git a/deploy/addons/metallb/metallb.yaml b/deploy/addons/metallb/metallb.yaml new file mode 100644 index 000000000000..0384232eab62 --- /dev/null +++ b/deploy/addons/metallb/metallb.yaml @@ -0,0 +1,293 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: metallb + name: metallb-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + fsGroup: + rule: RunAsAny + hostNetwork: true + hostPorts: + - max: 7472 + min: 7472 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - update +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port=7472 + - --config=config + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: metallb/speaker:v0.8.2 + imagePullPolicy: IfNotPresent + name: speaker + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - --port=7472 + - --config=config + image: metallb/controller:v0.8.2 + imagePullPolicy: IfNotPresent + name: controller + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + beta.kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 059ddf79294f..40863db42b54 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -122,4 +122,9 @@ var Addons = []*Addon{ set: SetBool, callbacks: []setFn{enableOrDisableStorageClasses}, }, + { + name: "metallb", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index b25446e8f668..0056883ba0e2 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -332,6 +332,20 @@ var Addons = map[string]*Addon{ "0640", false), }, false, "ingress-dns"), + "metallb": NewAddon([]*BinAsset{ + MustBinAsset( + "deploy/addons/metallb/metallb.yaml", + vmpath.GuestAddonsDir, + "metallb.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/metallb/metallb-config.yaml.tmpl", + vmpath.GuestAddonsDir, + "metallb-config.yaml", + "0640", + true), + }, false, "metallb"), } // GenerateTemplateData generates template data for template assets @@ -345,13 +359,17 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ea = "-" + runtime.GOARCH } opts := struct { - Arch string - ExoticArch string - ImageRepository string + Arch string + ExoticArch string + ImageRepository string + LoadBalancerStartIP string + LoadBalancerEndIP string }{ - Arch: a, - ExoticArch: ea, - ImageRepository: cfg.ImageRepository, + Arch: a, + ExoticArch: ea, + ImageRepository: cfg.ImageRepository, + LoadBalancerStartIP: cfg.LoadBalancerStartIP, + LoadBalancerEndIP: cfg.LoadBalancerEndIP, } return opts diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 6513efd2d99c..e9a9581689c8 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -71,19 +71,21 @@ type MachineConfig struct { // KubernetesConfig contains the parameters used to configure the VM Kubernetes. type KubernetesConfig struct { - KubernetesVersion string - ClusterName string - APIServerName string - APIServerNames []string - APIServerIPs []net.IP - DNSDomain string - ContainerRuntime string - CRISocket string - NetworkPlugin string - FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ - ServiceCIDR string // the subnet which kubernetes services will be deployed to - ImageRepository string - ExtraOptions ExtraOptionSlice + KubernetesVersion string + ClusterName string + APIServerName string + APIServerNames []string + APIServerIPs []net.IP + DNSDomain string + ContainerRuntime string + CRISocket string + NetworkPlugin string + FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + ServiceCIDR string // the subnet which kubernetes services will be deployed to + ImageRepository string + LoadBalancerStartIP string + LoadBalancerEndIP string + ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool EnableDefaultCNI bool From 377a4746f9d6dcb187a392f5e32422064ac7fe48 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 5 Mar 2020 01:41:45 +0800 Subject: [PATCH 2/5] Allows to configure load balancer start/end IP --- cmd/minikube/cmd/config/configure.go | 28 ++++++++++++++++++++++++++++ cmd/minikube/cmd/config/prompt.go | 20 ++++++++++++++++++++ cmd/minikube/cmd/start.go | 6 ++++++ 3 files changed, 54 insertions(+) diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 412fa1204d1f..5bcecc893ac5 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -18,8 +18,12 @@ package config import ( "io/ioutil" + "net" "github.com/spf13/cobra" + "github.com/spf13/viper" + + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/service" @@ -178,6 +182,30 @@ var addonsConfigureCmd = &cobra.Command{ out.WarningT("ERROR creating `registry-creds-acr` secret") } + case "metallb": + profile := viper.GetString(config.MachineProfile) + cfg, err := config.Load(profile) + if err != nil { + out.ErrT(out.FatalType, "Failed to load config {{.profile}}", out.V{"profile": profile}) + } + + validator := func(s string) bool { + return net.ParseIP(s) != nil + } + + if cfg.KubernetesConfig.LoadBalancerStartIP == "" { + cfg.KubernetesConfig.LoadBalancerStartIP = AskForStaticValidatedValue("-- Enter Load Balancer Start IP: ", validator) + } + + if cfg.KubernetesConfig.LoadBalancerEndIP == "" { + cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator) + } + + err = config.SaveProfile(profile, cfg) + if err != nil { + out.ErrT(out.FatalType, "Failed to save config {{.profile}}", out.V{"profile": profile}) + } + default: out.FailureT("{{.name}} has no available configuration options", out.V{"name": addon}) return diff --git a/cmd/minikube/cmd/config/prompt.go b/cmd/minikube/cmd/config/prompt.go index 10c00a7f0f51..8f85d6061c20 100644 --- a/cmd/minikube/cmd/config/prompt.go +++ b/cmd/minikube/cmd/config/prompt.go @@ -153,3 +153,23 @@ func posString(slice []string, element string) int { func containsString(slice []string, element string) bool { return posString(slice, element) != -1 } + +// AskForStaticValidatedValue asks for a single value to enter and check for valid input +func AskForStaticValidatedValue(s string, validator func(s string) bool) string { + reader := bufio.NewReader(os.Stdin) + + for { + response := getStaticValue(reader, s) + + // Can't have zero length + if len(response) == 0 { + out.Err("--Error, please enter a value:") + continue + } + if !validator(response) { + out.Err("--Invalid input, please enter a value:") + continue + } + return response + } +} diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index de5002a4c932..8990756d4bd9 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -118,6 +118,8 @@ const ( autoUpdate = "auto-update-drivers" hostOnlyNicType = "host-only-nic-type" natNicType = "nat-nic-type" + loadBalancerStartIP = "load-balancer-start-ip" + loadBalancerEndIP = "load-balancer-end-ip" ) var ( @@ -185,6 +187,8 @@ func initKubernetesFlags() { startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") + startCmd.Flags().String(loadBalancerStartIP, "", "The first Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") + startCmd.Flags().String(loadBalancerEndIP, "", "The last Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") } // initDriverFlags inits the commandline flags for vm drivers @@ -824,6 +828,8 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) ExtraOptions: node.ExtraOptions, ShouldLoadCachedImages: viper.GetBool(cacheImages), EnableDefaultCNI: selectedEnableDefaultCNI, + LoadBalancerStartIP: viper.GetString(loadBalancerStartIP), + LoadBalancerEndIP: viper.GetString(loadBalancerEndIP), }, Nodes: []config.Node{cp}, } From 30f08824180d71dd75b3e1dacac49760d53d6c65 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sun, 29 Mar 2020 11:45:53 +0800 Subject: [PATCH 3/5] Fixes MachineProfile no longer exist --- cmd/minikube/cmd/config/configure.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 5bcecc893ac5..3f361f3bbe5c 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -21,8 +21,6 @@ import ( "net" "github.com/spf13/cobra" - "github.com/spf13/viper" - "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" @@ -183,7 +181,7 @@ var addonsConfigureCmd = &cobra.Command{ } case "metallb": - profile := viper.GetString(config.MachineProfile) + profile := ClusterFlagValue() cfg, err := config.Load(profile) if err != nil { out.ErrT(out.FatalType, "Failed to load config {{.profile}}", out.V{"profile": profile}) From c57375c48626962d66ffd2494a45a2b8e65b0864 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sun, 29 Mar 2020 12:53:25 +0800 Subject: [PATCH 4/5] Remove load balancer start/end IP from start flags --- cmd/minikube/cmd/start.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index fcb32a9052a5..01f27f68c465 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -122,8 +122,6 @@ const ( autoUpdate = "auto-update-drivers" hostOnlyNicType = "host-only-nic-type" natNicType = "nat-nic-type" - loadBalancerStartIP = "load-balancer-start-ip" - loadBalancerEndIP = "load-balancer-end-ip" nodes = "nodes" ) @@ -193,8 +191,6 @@ func initKubernetesFlags() { startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") - startCmd.Flags().String(loadBalancerStartIP, "", "The first Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") - startCmd.Flags().String(loadBalancerEndIP, "", "The last Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") } // initDriverFlags inits the commandline flags for vm drivers @@ -1009,8 +1005,6 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor ExtraOptions: config.ExtraOptions, ShouldLoadCachedImages: viper.GetBool(cacheImages), EnableDefaultCNI: selectedEnableDefaultCNI, - LoadBalancerStartIP: viper.GetString(loadBalancerStartIP), - LoadBalancerEndIP: viper.GetString(loadBalancerEndIP), }, Nodes: []config.Node{cp}, } From f078cce376e1aa0be291f140a2232274af263576 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 9 Apr 2020 23:39:06 +0800 Subject: [PATCH 5/5] Add comment on LoadBalancerStartIP/LoadBalancerEndIP --- pkg/minikube/config/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index a7850a6ef5ed..3008f8b2ddd4 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -81,8 +81,8 @@ type KubernetesConfig struct { FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ ServiceCIDR string // the subnet which kubernetes services will be deployed to ImageRepository string - LoadBalancerStartIP string - LoadBalancerEndIP string + LoadBalancerStartIP string // currently only used by MetalLB addon + LoadBalancerEndIP string // currently only used by MetalLB addon ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool