Skip to content

Commit

Permalink
Get user-data/startupScripts from secrets (openshift#173)
Browse files Browse the repository at this point in the history
This commit adds the ability to pass custom userData/startupScripts to
the machines using kubernetes secrets. The previous workflow that used
the machine-setup ConfigMap has been removed in favor of the new one.

Fixes openshift#158
  • Loading branch information
flaper87 committed Jan 10, 2019
1 parent b0242cb commit 973889f
Show file tree
Hide file tree
Showing 13 changed files with 537 additions and 627 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ items:
floatingIP: 129.114.111.153
securityGroups:
- default
userDataSecret:
name: master-user-data
namespace: openstack-provider-system
versions:
kubelet: 1.12.3
controlPlane: 1.12.3
Expand All @@ -42,5 +45,8 @@ items:
floatingIP: 129.114.111.153
securityGroups:
- default
userDataSecret:
name: worker-user-data
namespace: openstack-provider-system
versions:
kubelet: 1.12.3
111 changes: 111 additions & 0 deletions cmd/clusterctl/examples/openstack/centos/master-user-data.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash
set -e
set -x
(
KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }}
VERSION=v${KUBELET_VERSION}
NAMESPACE={{ .Machine.ObjectMeta.Namespace }}
MACHINE=$NAMESPACE
MACHINE+="/"
MACHINE+={{ .Machine.ObjectMeta.Name }}
CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }}
CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }}
POD_CIDR={{ .PodCIDR }}
SERVICE_CIDR={{ .ServiceCIDR }}
ARCH=amd64
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kube*
EOF

setenforce 0
yum install -y kubelet-$CONTROL_PLANE_VERSION kubeadm-$CONTROL_PLANE_VERSION kubectl-$CONTROL_PLANE_VERSION --disableexcludes=kubernetes

function install_configure_docker () {
# prevent docker from auto-starting
echo "exit 101" > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
trap "rm /usr/sbin/policy-rc.d" RETURN
yum install -y docker
echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
systemctl daemon-reload
systemctl enable docker
systemctl start docker
}

install_configure_docker

cat <<EOF > /etc/default/kubelet
KUBELET_KUBEADM_EXTRA_ARGS=--cgroup-driver=systemd
EOF

systemctl enable kubelet.service

modprobe br_netfilter
echo '1' > /proc/sys/net/bridge/bridge-nf-call-iptables
echo '1' > /proc/sys/net/ipv4/ip_forward

echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf

# Set up kubeadm config file to pass parameters to kubeadm init.
cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
bootstrapTokens:
- token: ${TOKEN}
nodeRegistration:
kubeletExtraArgs:
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
---
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v${CONTROL_PLANE_VERSION}
networking:
serviceSubnet: ${SERVICE_CIDR}
clusterName: kubernetes
apiServerExtraArgs:
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
apiServerExtraVolumes:
- name: cloud
hostPath: "/etc/kubernetes/cloud.conf"
mountPath: "/etc/kubernetes/cloud.conf"
controlPlaneEndpoint: ""
controllerManagerExtraArgs:
cluster-cidr: ${POD_CIDR}
service-cluster-ip-range: ${SERVICE_CIDR}
allocate-node-cidrs: "true"
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
controllerManagerExtraVolumes:
- name: cloud
hostPath: "/etc/kubernetes/cloud.conf"
mountPath: "/etc/kubernetes/cloud.conf"
EOF

kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
# Enable networking by default.
kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml --kubeconfig /etc/kubernetes/admin.conf

# By default, use calico for container network plugin, should make this configurable.
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

mkdir -p /root/.kube
cp -i /etc/kubernetes/admin.conf /root/.kube/config
chown $(id -u):$(id -g) /root/.kube/config

echo done.
) 2>&1 | tee /var/log/startup.log

Original file line number Diff line number Diff line change
Expand Up @@ -19,204 +19,19 @@ data:
clouds.yaml: $OPENSTACK_CLOUD_CONFIG
---
apiVersion: v1
kind: ConfigMap
kind: Secret
type: Opaque
metadata:
name: worker-user-data
namespace: openstack-provider-system
data:
userData: $WORKER_USER_DATA
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: machine-setup
name: master-user-data
namespace: openstack-provider-system
data:
machine_setup_configs.yaml: |-
items:
- versions:
kubelet: 1.12.3
controlPlane: 1.12.3
startupScript: |
#!/bin/bash
set -e
set -x
(
KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }}
VERSION=v${KUBELET_VERSION}
NAMESPACE={{ .Machine.ObjectMeta.Namespace }}
MACHINE=$NAMESPACE
MACHINE+="/"
MACHINE+={{ .Machine.ObjectMeta.Name }}
CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }}
CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }}
POD_CIDR={{ .PodCIDR }}
SERVICE_CIDR={{ .ServiceCIDR }}
ARCH=amd64
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kube*
EOF

setenforce 0
yum install -y kubelet-$CONTROL_PLANE_VERSION kubeadm-$CONTROL_PLANE_VERSION kubectl-$CONTROL_PLANE_VERSION --disableexcludes=kubernetes

function install_configure_docker () {
# prevent docker from auto-starting
echo "exit 101" > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
trap "rm /usr/sbin/policy-rc.d" RETURN
yum install -y docker
echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
systemctl daemon-reload
systemctl enable docker
systemctl start docker
}

install_configure_docker

cat <<EOF > /etc/default/kubelet
KUBELET_KUBEADM_EXTRA_ARGS=--cgroup-driver=systemd
EOF

systemctl enable kubelet.service

modprobe br_netfilter
echo '1' > /proc/sys/net/bridge/bridge-nf-call-iptables
echo '1' > /proc/sys/net/ipv4/ip_forward

echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf

# Set up kubeadm config file to pass parameters to kubeadm init.
cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
bootstrapTokens:
- token: ${TOKEN}
nodeRegistration:
kubeletExtraArgs:
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
---
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v${CONTROL_PLANE_VERSION}
networking:
serviceSubnet: ${SERVICE_CIDR}
clusterName: kubernetes
apiServerExtraArgs:
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
apiServerExtraVolumes:
- name: cloud
hostPath: "/etc/kubernetes/cloud.conf"
mountPath: "/etc/kubernetes/cloud.conf"
controlPlaneEndpoint: ""
controllerManagerExtraArgs:
cluster-cidr: ${POD_CIDR}
service-cluster-ip-range: ${SERVICE_CIDR}
allocate-node-cidrs: "true"
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
controllerManagerExtraVolumes:
- name: cloud
hostPath: "/etc/kubernetes/cloud.conf"
mountPath: "/etc/kubernetes/cloud.conf"
EOF

kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
# Enable networking by default.
kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml --kubeconfig /etc/kubernetes/admin.conf

# By default, use calico for container network plugin, should make this configurable.
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

mkdir -p /root/.kube
cp -i /etc/kubernetes/admin.conf /root/.kube/config
chown $(id -u):$(id -g) /root/.kube/config

echo done.
) 2>&1 | tee /var/log/startup.log
- versions:
kubelet: 1.12.3
startupScript: |
#!/bin/bash
set -e
set -x
(
KUBELET_VERSION={{ .Machine.Spec.Versions.Kubelet }}
TOKEN={{ .Token }}
MASTER={{ call .GetMasterEndpoint }}
NAMESPACE={{ .Machine.ObjectMeta.Namespace }}
MACHINE=$NAMESPACE
MACHINE+="/"
MACHINE+={{ .Machine.ObjectMeta.Name }}
CLUSTER_DNS_DOMAIN={{ .Cluster.Spec.ClusterNetwork.ServiceDomain }}
POD_CIDR={{ .PodCIDR }}
SERVICE_CIDR={{ .ServiceCIDR }}
CONTROL_PLANE_VERSION={{ .Machine.Spec.Versions.ControlPlane }}
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kube*
EOF

setenforce 0
yum install -y kubelet-$CONTROL_PLANE_VERSION kubeadm-$CONTROL_PLANE_VERSION kubectl-$CONTROL_PLANE_VERSION --disableexcludes=kubernetes

function install_configure_docker () {
# prevent docker from auto-starting
echo "exit 101" > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
trap "rm /usr/sbin/policy-rc.d" RETURN
yum install -y docker
echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
systemctl daemon-reload
systemctl enable docker
systemctl start docker
}

install_configure_docker

# Write the cloud.conf so that the kubelet can use it.
echo $OPENSTACK_CLOUD_PROVIDER_CONF | base64 -d > /etc/kubernetes/cloud.conf

# Set up kubeadm config file to pass to kubeadm join.
cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1alpha3
kind: JoinConfiguration
nodeRegistration:
kubeletExtraArgs:
cloud-provider: "openstack"
cloud-config: "/etc/kubernetes/cloud.conf"
token: ${TOKEN}
discoveryTokenAPIServers:
- ${MASTER}
discoveryTokenUnsafeSkipCAVerification: true
EOF

cat <<EOF > /etc/default/kubelet
KUBELET_KUBEADM_EXTRA_ARGS=--cgroup-driver=systemd
EOF
systemctl enable kubelet.service

modprobe br_netfilter
echo '1' > /proc/sys/net/bridge/bridge-nf-call-iptables
echo '1' > /proc/sys/net/ipv4/ip_forward

kubeadm join --ignore-preflight-errors=all --config /etc/kubernetes/kubeadm_config.yaml
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done

echo done.
) 2>&1 | tee /var/log/startup.log
userData: $MASTER_USER_DATA
Loading

0 comments on commit 973889f

Please sign in to comment.