Skip to content

Commit

Permalink
Add user-configurable images
Browse files Browse the repository at this point in the history 
 Refactor configuration defaults
 Add comments to config
 Add configurable utility images
 Add configurable network plugin images
 Add configurable kubedns images
  • Loading branch information
@moelsayed
moelsayed committed Dec 5, 2017
1 parent 89ec803 commit 0ee002b
Show file tree
Hide file tree
Showing 19 changed files with 278 additions and 135 deletions.
29 changes: 19 additions & 10 deletions addons/kubedns.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
package addons

func GetKubeDNSManifest(clusterDNSServer, clusterDomain string) string {
const (
KubeDNSImage = "kubeDNSImage"
DNSMasqImage = "DNSMasqImage"
KubeDNSSidecarImage = "kubednsSidecarImage"
KubeDNSAutoScalerImage = "kubeDNSAutoScalerImage"
KubeDNSServer = "clusterDNSServer"
KubeDNSClusterDomain = "clusterDomain"
)

func GetKubeDNSManifest(kubeDNSConfig map[string]string) string {
return `---
apiVersion: apps/v1beta1
kind: Deployment
Expand All @@ -17,7 +26,7 @@ spec:
spec:
containers:
- name: autoscaler
image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0
image: ` + kubeDNSConfig[KubeDNSAutoScalerImage] + `
resources:
requests:
cpu: "20m"
Expand Down Expand Up @@ -81,7 +90,7 @@ spec:
optional: true
containers:
- name: kubedns
image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5
image: ` + kubeDNSConfig[KubeDNSImage] + `
resources:
# TODO: Set memory limits when we've profiled the container for large
# clusters, then set request = limit to keep this container in
Expand Down Expand Up @@ -111,7 +120,7 @@ spec:
initialDelaySeconds: 3
timeoutSeconds: 5
args:
- --domain=` + clusterDomain + `.
- --domain=` + kubeDNSConfig[KubeDNSClusterDomain] + `.
- --dns-port=10053
- --config-dir=/kube-dns-config
- --v=2
Expand All @@ -132,7 +141,7 @@ spec:
- name: kube-dns-config
mountPath: /kube-dns-config
- name: dnsmasq
image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5
image: ` + kubeDNSConfig[DNSMasqImage] + `
livenessProbe:
httpGet:
path: /healthcheck/dnsmasq
Expand All @@ -151,7 +160,7 @@ spec:
- -k
- --cache-size=1000
- --log-facility=-
- --server=/` + clusterDomain + `/127.0.0.1#10053
- --server=/` + kubeDNSConfig[KubeDNSClusterDomain] + `/127.0.0.1#10053
- --server=/in-addr.arpa/127.0.0.1#10053
- --server=/ip6.arpa/127.0.0.1#10053
ports:
Expand All @@ -170,7 +179,7 @@ spec:
- name: kube-dns-config
mountPath: /etc/k8s/dns/dnsmasq-nanny
- name: sidecar
image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5
image: ` + kubeDNSConfig[KubeDNSSidecarImage] + `
livenessProbe:
httpGet:
path: /metrics
Expand All @@ -183,8 +192,8 @@ spec:
args:
- --v=2
- --logtostderr
- --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.` + clusterDomain + `,5,A
- --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.` + clusterDomain + `,5,A
- --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.` + kubeDNSConfig[KubeDNSClusterDomain] + `,5,A
- --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.` + kubeDNSConfig[KubeDNSClusterDomain] + `,5,A
ports:
- containerPort: 10054
name: metrics
Expand All @@ -209,7 +218,7 @@ metadata:
spec:
selector:
k8s-app: kube-dns
clusterIP: ` + clusterDNSServer + `
clusterIP: ` + kubeDNSConfig[KubeDNSServer] + `
ports:
- name: dns
port: 53
Expand Down
13 changes: 12 additions & 1 deletion cluster.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ auth:
network:
plugin: flannel
options:
foo: bar
flannel_image: quay.io/coreos/flannel:v0.9.1
flannel_cni_image: quay.io/coreos/flannel-cni:v0.2.0

ssh_key_path: ~/.ssh/test

Expand Down Expand Up @@ -56,6 +57,16 @@ services:
kubeproxy:
image: rancher/k8s:v1.8.3-rancher2


system_images:
alpine: alpine:latest
nginx_proxy: rancher/rke-nginx-proxy:0.1.0
cert_downloader: rancher/rke-cert-deployer:0.1.0
kubedns_image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5
dnsmasq_image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5
kubedns_sidecar_image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5
kubedns_autoscaler_image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0

# all addon manifests MUST specify a namespace
addons: |-
---
Expand Down
12 changes: 9 additions & 3 deletions cluster/addons.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,15 @@ func (c *Cluster) DeployUserAddOns() error {

func (c *Cluster) deployKubeDNS() error {
logrus.Infof("[addons] Setting up KubeDNS")

kubeDNSYaml := addons.GetKubeDNSManifest(c.ClusterDNSServer, c.ClusterDomain)

kubeDNSConfig := map[string]string{
addons.KubeDNSServer: c.ClusterDNSServer,
addons.KubeDNSClusterDomain: c.ClusterDomain,
addons.KubeDNSImage: c.SystemImages[KubeDNSImage],
addons.DNSMasqImage: c.SystemImages[DNSMasqImage],
addons.KubeDNSSidecarImage: c.SystemImages[KubeDNSSidecarImage],
addons.KubeDNSAutoScalerImage: c.SystemImages[KubeDNSAutoScalerImage],
}
kubeDNSYaml := addons.GetKubeDNSManifest(kubeDNSConfig)
if err := c.doAddonDeploy(kubeDNSYaml, KubeDNSAddonResourceName); err != nil {
return err
}
Expand Down
83 changes: 47 additions & 36 deletions cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,18 @@ type Cluster struct {
}

const (
X509AuthenticationProvider = "x509"
DefaultClusterConfig = "cluster.yml"
DefaultServiceClusterIPRange = "10.233.0.0/18"
DefaultClusterCIDR = "10.233.64.0/18"
DefaultClusterDNSService = "10.233.0.3"
DefaultClusterDomain = "cluster.local"
DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0"
DefaultAuthStrategy = "x509"
DefaultNetworkPlugin = "flannel"
DefaultClusterSSHKeyPath = "~/.ssh/id_rsa"
StateConfigMapName = "cluster-state"
UpdateStateTimeout = 30
GetStateTimeout = 30
KubernetesClientTimeOut = 30
X509AuthenticationProvider = "x509"
StateConfigMapName = "cluster-state"
UpdateStateTimeout = 30
GetStateTimeout = 30
KubernetesClientTimeOut = 30
AplineImage = "alpine"
NginxProxyImage = "nginx_proxy"
CertDownloaderImage = "cert_downloader"
KubeDNSImage = "kubedns_image"
DNSMasqImage = "dnsmasq_image"
KubeDNSSidecarImage = "kubedns_sidecar_image"
KubeDNSAutoScalerImage = "kubedns_autoscaler_image"
)

func (c *Cluster) DeployClusterPlanes() error {
Expand All @@ -59,7 +57,7 @@ func (c *Cluster) DeployClusterPlanes() error {
if err != nil {
return fmt.Errorf("[controlPlane] Failed to bring up Control Plane: %v", err)
}
err = services.RunWorkerPlane(c.ControlPlaneHosts, c.WorkerHosts, c.Services)
err = services.RunWorkerPlane(c.ControlPlaneHosts, c.WorkerHosts, c.Services, c.SystemImages[NginxProxyImage])
if err != nil {
return fmt.Errorf("[workerPlane] Failed to bring up Worker Plane: %v", err)
}
Expand Down Expand Up @@ -126,29 +124,42 @@ func (c *Cluster) setClusterDefaults() {
c.Nodes[i].SSHKeyPath = c.SSHKeyPath
}
}
if len(c.Services.KubeAPI.ServiceClusterIPRange) == 0 {
c.Services.KubeAPI.ServiceClusterIPRange = DefaultServiceClusterIPRange
}
if len(c.Services.KubeController.ServiceClusterIPRange) == 0 {
c.Services.KubeController.ServiceClusterIPRange = DefaultServiceClusterIPRange
}
if len(c.Services.KubeController.ClusterCIDR) == 0 {
c.Services.KubeController.ClusterCIDR = DefaultClusterCIDR
}
if len(c.Services.Kubelet.ClusterDNSServer) == 0 {
c.Services.Kubelet.ClusterDNSServer = DefaultClusterDNSService
}
if len(c.Services.Kubelet.ClusterDomain) == 0 {
c.Services.Kubelet.ClusterDomain = DefaultClusterDomain
}
if len(c.Services.Kubelet.InfraContainerImage) == 0 {
c.Services.Kubelet.InfraContainerImage = DefaultInfraContainerImage
c.setClusterServicesDefaults()
c.setClusterNetworkDefaults()
c.setClusterImageDefaults()
}

func (c *Cluster) setClusterServicesDefaults() {
serviceConfigDefaultsMap := map[string]string{
c.Services.KubeAPI.ServiceClusterIPRange: DefaultServiceClusterIPRange,
c.Services.KubeController.ServiceClusterIPRange: DefaultServiceClusterIPRange,
c.Services.KubeController.ClusterCIDR: DefaultClusterCIDR,
c.Services.Kubelet.ClusterDNSServer: DefaultClusterDNSService,
c.Services.Kubelet.ClusterDomain: DefaultClusterDomain,
c.Services.Kubelet.InfraContainerImage: DefaultInfraContainerImage,
c.Authentication.Strategy: DefaultAuthStrategy,
}
if len(c.Authentication.Strategy) == 0 {
c.Authentication.Strategy = DefaultAuthStrategy
for k, v := range serviceConfigDefaultsMap {
setDefaultIfEmpty(&k, v)
}
if len(c.Network.Plugin) == 0 {
c.Network.Plugin = DefaultNetworkPlugin
}

func (c *Cluster) setClusterImageDefaults() {
if c.SystemImages == nil {
// don't break if the user didn't define rke_images
c.SystemImages = make(map[string]string)
}
systemImagesDefaultsMap := map[string]string{
AplineImage: DefaultAplineImage,
NginxProxyImage: DefaultNginxProxyImage,
CertDownloaderImage: DefaultCertDownloaderImage,
KubeDNSImage: DefaultKubeDNSImage,
DNSMasqImage: DefaultDNSMasqImage,
KubeDNSSidecarImage: DefaultKubeDNSSidecarImage,
KubeDNSAutoScalerImage: DefaultKubeDNSAutoScalerImage,
}
for k, v := range systemImagesDefaultsMap {
setDefaultIfEmptyMapValue(c.SystemImages, k, v)
}
}

Expand Down
45 changes: 45 additions & 0 deletions cluster/defaults.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package cluster

const (
DefaultClusterConfig = "cluster.yml"

DefaultServiceClusterIPRange = "10.233.0.0/18"
DefaultClusterCIDR = "10.233.64.0/18"
DefaultClusterDNSService = "10.233.0.3"
DefaultClusterDomain = "cluster.local"
DefaultClusterSSHKeyPath = "~/.ssh/id_rsa"

DefaultAuthStrategy = "x509"

DefaultNetworkPlugin = "flannel"

DefaultInfraContainerImage = "gcr.io/google_containers/pause-amd64:3.0"
DefaultAplineImage = "alpine:latest"
DefaultNginxProxyImage = "rancher/rke-nginx-proxy:0.1.0"
DefaultCertDownloaderImage = "rancher/rke-cert-deployer:0.1.0"

DefaultFlannelImage = "quay.io/coreos/flannel:v0.9.1"
DefaultFlannelCNIImage = "quay.io/coreos/flannel-cni:v0.2.0"
DefaultCalicoNodeImage = "quay.io/calico/node:v2.6.2"
DefaultCalicoCNIImage = "quay.io/calico/cni:v1.11.0"
DefaultCalicoControllersImage = "quay.io/calico/kube-controllers:v1.0.0"
DefaultCanalNodeImage = "quay.io/calico/node:v2.6.2"
DefaultCanalCNIImage = "quay.io/calico/cni:v1.11.0"
DefaultCanalFlannelImage = "quay.io/coreos/flannel:v0.9.1"

DefaultKubeDNSImage = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5"
DefaultDNSMasqImage = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
DefaultKubeDNSSidecarImage = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
DefaultKubeDNSAutoScalerImage = "gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.0.0"
)

func setDefaultIfEmptyMapValue(configMap map[string]string, key string, value string) {
if _, ok := configMap[key]; !ok {
configMap[key] = value
}
}
func setDefaultIfEmpty(varName *string, defaultValue string) {
if len(*varName) == 0 {
*varName = defaultValue
}
}
4 changes: 2 additions & 2 deletions cluster/hosts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ func (c *Cluster) InvertIndexHosts() error {
func (c *Cluster) SetUpHosts() error {
if c.Authentication.Strategy == X509AuthenticationProvider {
logrus.Infof("[certificates] Deploying kubernetes certificates to Cluster nodes")
err := pki.DeployCertificatesOnMasters(c.ControlPlaneHosts, c.Certificates)
err := pki.DeployCertificatesOnMasters(c.ControlPlaneHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
if err != nil {
return err
}
err = pki.DeployCertificatesOnWorkers(c.WorkerHosts, c.Certificates)
err = pki.DeployCertificatesOnWorkers(c.WorkerHosts, c.Certificates, c.SystemImages[CertDownloaderImage])
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 0ee002b

Please sign in to comment.