diff --git a/images/Makefile b/images/Makefile index d2554d953..ca5b3950c 100644 --- a/images/Makefile +++ b/images/Makefile @@ -84,7 +84,6 @@ build-all: $(MAKE) ${OP} WHAT=ubuntu RELEASE=16.04 IS_MANIFEST_LIST=0 $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=arm64 $(MAKE) ${OP} WHAT=ubuntu RELEASE=18.04 IS_MANIFEST_LIST=1 GOARCH=amd64 - $(MAKE) ${OP} WHAT=ubuntu RELEASE=19.04 IS_MANIFEST_LIST=0 $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=arm64 $(MAKE) ${OP} WHAT=ubuntu RELEASE=20.04 IS_LATEST=true IS_MANIFEST_LIST=1 GOARCH=amd64 $(MAKE) ${OP} WHAT=centos RELEASE=7 diff --git a/images/centos/Dockerfile b/images/centos/Dockerfile index 786b9624e..5b972131b 100644 --- a/images/centos/Dockerfile +++ b/images/centos/Dockerfile @@ -14,4 +14,4 @@ RUN yum -y install \ yum clean all # Set the root password to root when logging in through the VM's ttyS0 console -RUN echo root | passwd --stdin root +RUN echo "root:root" | chpasswd diff --git a/images/kubeadm/.dockerignore b/images/kubeadm/.dockerignore new file mode 100644 index 000000000..f5bdd214e --- /dev/null +++ b/images/kubeadm/.dockerignore @@ -0,0 +1 @@ +run diff --git a/images/kubeadm/Dockerfile b/images/kubeadm/Dockerfile index 327037603..8b4289104 100644 --- a/images/kubeadm/Dockerfile +++ b/images/kubeadm/Dockerfile @@ -1,4 +1,5 @@ -FROM weaveworks/ignite-ubuntu:20.04 +# Ubuntu 20.04 was also tested, but didn't perform very well (sshd took a long time to start), so we're sticking with Ubuntu 18.04 still +FROM weaveworks/ignite-ubuntu:18.04 # Install dependencies. Use containerd for running the containers (for better performance) RUN apt-get update && apt-get install -y --no-install-recommends \ apt-transport-https \ diff --git a/images/kubeadm/README.md b/images/kubeadm/README.md index 3ea485136..3fa9f8b13 100644 --- a/images/kubeadm/README.md +++ b/images/kubeadm/README.md @@ -1,12 +1,10 @@ -## Run kubeadm in HA mode with Ignite VMs +# Run kubeadm in HA mode with Ignite VMs This short guide shows you how to setup Kubernetes in HA mode with Ignite VMs. **NOTE:** At the moment, you need to execute all these commands as `root`. -**NOTE:** This guide assumes you have no running containers, in other words, that -the IP of the first docker container that will be run is `172.17.0.2`. You can check -this with `docker run --rm busybox ip addr`. +**NOTE:** It is assumed that you start no new VMs between running `prepare.sh` and starting the masters, as IP addresses are computed consecutively First set up some files and certificates using `prepare.sh` from this directory: @@ -16,7 +14,7 @@ First set up some files and certificates using `prepare.sh` from this directory: This will create a kubeadm configuration file, generate the CA cert, give you a kubeconfig file, etc. -### Start the seed master +## Start the seed master For the bootstap master, copy over the CA cert and key to use, and the kubeadm config file: @@ -29,15 +27,18 @@ ignite run weaveworks/ignite-kubeadm:latest \ --copy-files $(pwd)/run/pki/ca.crt:/etc/kubernetes/pki/ca.crt \ --copy-files $(pwd)/run/pki/ca.key:/etc/kubernetes/pki/ca.key \ --name master-0 + +# Get the IP address of the initial master, for the kubeadm join command below +export MASTER_IP=$($ignite inspect vm master-0 | jq -r ".status.ipAddresses[0]") ``` Initialize it with `kubeadm` using `ignite exec`: ```bash -ignite exec master-0 kubeadm init --config /kubeadm.yaml --upload-certs +ignite exec master-0 -- kubeadm init --config /kubeadm.yaml --upload-certs ``` -### Join additional masters +## Join additional masters Create more master VMs, but copy only the variables we need for joining: @@ -47,7 +48,6 @@ for i in {1..2}; do --cpus 2 \ --memory 1GB \ --ssh \ - --copy-files $(pwd)/run/k8s-vars.sh:/etc/profile.d/02-k8s.sh \ --name master-${i} done ``` @@ -55,8 +55,9 @@ done Use `ignite exec` to join each VM to the control plane: ```bash +source run/k8s-vars.sh for i in {1..2}; do - ignite exec master-${i} kubeadm join firekube.luxas.dev:6443 \ + ignite exec master-${i} -- kubeadm join ${MASTER_IP}.xip.io:6443 \ --token ${TOKEN} \ --discovery-token-ca-cert-hash sha256:${CA_HASH} \ --certificate-key ${CERT_KEY} \ @@ -64,13 +65,13 @@ for i in {1..2}; do done ``` -### Set up a HAProxy loadbalancer locally +## Set up a HAProxy loadbalancer locally ```bash -docker run -d -v $(pwd)/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg -p 6443:443 haproxy:alpine +docker run -d -v $(pwd)/run/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg -p 6443:443 haproxy:alpine ``` -### Use kubectl +## Use kubectl This will make `kubectl` talk to any of the three masters you've set up, via HAproxy. @@ -82,17 +83,17 @@ kubectl get nodes Right now it's expected that the nodes are in state `NotReady`, as CNI networking isn't set up. -#### Install a CNI Network -- Weave Net +### Install a CNI Network -- Weave Net We're going to use [Weave Net](https://github.com/weaveworks/weave). ```bash -kubectl apply -f https://git.io/weave-kube-1.6 +kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" ``` With this, the nodes should transition into the `Ready` state in a minute or so. -### Watch the cluster heal +## Watch the cluster heal Kill the bootstrap master and see the cluster recover: @@ -104,4 +105,4 @@ kubectl get nodes What's happening underneath here is that HAproxy (or any other loadbalancer) notices that `master-0` is unhealthy, and removes it from the roundrobin list. etcd also realizes -that one peer is lost, and re-elects a leader amongst the two that are still standing. \ No newline at end of file +that one peer is lost, and re-elects a leader amongst the two that are still standing. diff --git a/images/kubeadm/haproxy.cfg b/images/kubeadm/haproxy.cfg deleted file mode 100644 index b4a5483e8..000000000 --- a/images/kubeadm/haproxy.cfg +++ /dev/null @@ -1,12 +0,0 @@ -frontend http_front - bind *:443 - stats uri /haproxy?stats - default_backend http_back - -backend http_back - balance roundrobin - option httpchk GET /healthz - http-check expect string ok - server master1 172.17.0.2:6443 check check-ssl verify none - server master2 172.17.0.3:6443 check check-ssl verify none - server master3 172.17.0.4:6443 check check-ssl verify none diff --git a/images/kubeadm/prepare.sh b/images/kubeadm/prepare.sh index fbca2ee9e..6b9646ae4 100755 --- a/images/kubeadm/prepare.sh +++ b/images/kubeadm/prepare.sh @@ -1,18 +1,35 @@ #!/bin/bash # Set up the seed node with the specified config file +IMAGE=${IMAGE:-"weaveworks/ignite-kubeadm"} + mkdir -p run -docker run -i --rm -v $(pwd)/run:/etc/kubernetes weaveworks/ignite-kubeadm \ +docker run -i --rm \ + -u $(id -u):$(id -g) \ + -v $(pwd)/run:/etc/kubernetes \ + ${IMAGE} \ kubeadm init phase certs ca -docker run -i --rm --net host -v $(pwd)/run:/etc/kubernetes weaveworks/ignite-kubeadm \ +docker run -i --rm \ + --net host \ + -u $(id -u):$(id -g) \ + -v $(pwd)/run:/etc/kubernetes \ + ${IMAGE} \ kubeadm init phase kubeconfig admin export HOST_IP=$(grep server run/admin.conf | grep -o -e "[0-9\.]*" | head -1) -export TOKEN=$(docker run -i --rm -v $(pwd)/run:/etc/kubernetes weaveworks/ignite-kubeadm kubeadm token generate) -export CERT_KEY=$(docker run -i --rm -v $(pwd)/run:/etc/kubernetes weaveworks/ignite-kubeadm kubeadm alpha certs certificate-key) +export TOKEN=$(docker run -i --rm -v $(pwd)/run:/etc/kubernetes ${IMAGE} kubeadm token generate) +export CERT_KEY=$(docker run -i --rm -v $(pwd)/run:/etc/kubernetes ${IMAGE} kubeadm alpha certs certificate-key) export CA_HASH=$(openssl x509 -pubkey -in run/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') +export LAST_ALLOCATED_IP=$(cat /var/lib/cni/networks/ignite-cni-bridge/last_reserved_ip.0) +export IP_PREFIX=$(echo ${LAST_ALLOCATED_IP} | cut -f-3 -d.) +export IP_START_NUMBER=$(echo ${LAST_ALLOCATED_IP} | cut -f4- -d.) + +export MASTER1_IP="${IP_PREFIX}.$((IP_START_NUMBER + 1))" +export MASTER2_IP="${IP_PREFIX}.$((IP_START_NUMBER + 2))" +export MASTER3_IP="${IP_PREFIX}.$((IP_START_NUMBER + 3))" + cat > run/config.yaml < run/haproxy.cfg <