diff --git a/hack/quickstart/init-master.sh b/hack/quickstart/init-master.sh index 4a039bb3e..a0c668507 100755 --- a/hack/quickstart/init-master.sh +++ b/hack/quickstart/init-master.sh @@ -7,6 +7,7 @@ REMOTE_USER=${REMOTE_USER:-core} CLUSTER_DIR=${CLUSTER_DIR:-cluster} IDENT=${IDENT:-${HOME}/.ssh/id_rsa} SSH_OPTS=${SSH_OPTS:-} +MULTI_MASTER=${MULTI_MASTER:-false} SELF_HOST_ETCD=${SELF_HOST_ETCD:-false} CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY:-false} CLOUD_PROVIDER=${CLOUD_PROVIDER:-} @@ -68,7 +69,7 @@ function init_master_node() { # Render cluster assets /home/${REMOTE_USER}/bootkube render --asset-dir=/home/${REMOTE_USER}/assets ${etcd_render_flags} ${cnp_render_flags} \ - --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 + --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443,https://127.0.0.1:6443 # Move the local kubeconfig into expected location chown -R ${REMOTE_USER}:${REMOTE_USER} /home/${REMOTE_USER}/assets @@ -76,6 +77,15 @@ function init_master_node() { cp /home/${REMOTE_USER}/assets/auth/kubeconfig /etc/kubernetes/ cp /home/${REMOTE_USER}/assets/tls/ca.crt /etc/kubernetes/ca.crt + if [ "$MULTI_MASTER" = true ]; then + sed -r -e "s/(server: ).*/\1https:\/\/127.0.0.1:6443/" -i /etc/kubernetes/kubeconfig + mkdir -p /etc/nginx + cp nginx.conf /etc/nginx + echo "server 127.0.0.1:443 backup;" > /etc/nginx/upstream.conf + mkdir -p /etc/kubernetes/manifests + cp nginx-proxy.yaml /etc/kubernetes/manifests + fi + # Start etcd. if [ "$SELF_HOST_ETCD" = false ] ; then configure_etcd @@ -119,16 +129,22 @@ if [ "${REMOTE_HOST}" != "local" ]; then exit 1 fi fi + + if [ "$MULTI_MASTER" = true ]; then + scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx.conf + scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx-proxy.yaml ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx-proxy.yaml + fi + # Copy self to remote host so script can be executed in "local" mode scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/init-master.sh - ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} SELF_HOST_ETCD=${SELF_HOST_ETCD} CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY} /home/${REMOTE_USER}/init-master.sh local" + ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} MULTI_MASTER=${MULTI_MASTER} SELF_HOST_ETCD=${SELF_HOST_ETCD} CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY} /home/${REMOTE_USER}/init-master.sh local" # Copy assets from remote host to a local directory. These can be used to launch additional nodes & contain TLS assets mkdir ${CLUSTER_DIR} scp -q -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} -r ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/assets/* ${CLUSTER_DIR} # Cleanup - ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm -rf /home/${REMOTE_USER}/{assets,init-master.sh,bootkube}" + ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm -rf /home/${REMOTE_USER}/{assets,init-master.sh,bootkube,nginx.conf,nginx-proxy.yaml}" echo "Cluster assets copied to ${CLUSTER_DIR}" echo diff --git a/hack/quickstart/init-node.sh b/hack/quickstart/init-node.sh index 82e181fbb..0e426eeda 100755 --- a/hack/quickstart/init-node.sh +++ b/hack/quickstart/init-node.sh @@ -8,6 +8,7 @@ REMOTE_USER=${REMOTE_USER:-core} IDENT=${IDENT:-${HOME}/.ssh/id_rsa} SSH_OPTS=${SSH_OPTS:-} TAG_MASTER=${TAG_MASTER:-false} +MULTI_MASTER=${MULTI_MASTER:-false} CLOUD_PROVIDER=${CLOUD_PROVIDER:-} function usage() { @@ -22,6 +23,14 @@ function init_worker_node() { # Setup kubeconfig mkdir -p /etc/kubernetes cp ${KUBECONFIG} /etc/kubernetes/kubeconfig + if [ "$MULTI_MASTER" = true ]; then + sed -r -e "s/(server: ).*/\1https:\/\/127.0.0.1:6443/" -i /etc/kubernetes/kubeconfig + mkdir -p /etc/nginx + cp nginx.conf /etc/nginx + cp upstream.conf /etc/nginx + mkdir -p /etc/kubernetes/manifests + cp nginx-proxy.yaml /etc/kubernetes/manifests + fi # Pulled out of the kubeconfig. Other installations should place the root # CA here manually. grep 'certificate-authority-data' ${KUBECONFIG} | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt @@ -51,12 +60,19 @@ if [ "${REMOTE_HOST}" != "local" ]; then fi scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${KUBECONFIG} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/kubeconfig + if [ "$MULTI_MASTER" = true ]; then + kubectl --kubeconfig=${KUBECONFIG} get node -lnode-role.kubernetes.io/master -o go-template --template '{{range .items}}{{"server "}}{{.metadata.name}}{{" backup;\n"}}{{end}}' > upstream.conf + scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx.conf + scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} upstream.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/upstream.conf + scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx-proxy.yaml ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx-proxy.yaml + fi + # Copy self to remote host so script can be executed in "local" mode scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/init-node.sh - ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} /home/${REMOTE_USER}/init-node.sh local /home/${REMOTE_USER}/kubeconfig" + ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} MULTI_MASTER=${MULTI_MASTER} CLOUD_PROVIDER=${CLOUD_PROVIDER} /home/${REMOTE_USER}/init-node.sh local /home/${REMOTE_USER}/kubeconfig" # Cleanup - ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm /home/${REMOTE_USER}/{init-node.sh,kubeconfig}" + ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm /home/${REMOTE_USER}/{init-node.sh,kubeconfig,nginx-proxy.yaml,nginx.conf}" echo echo "Node bootstrap complete. It may take a few minutes for the node to become ready. Access your kubernetes cluster using:" diff --git a/hack/quickstart/nginx-proxy.yaml b/hack/quickstart/nginx-proxy.yaml new file mode 100644 index 000000000..ad7a45a80 --- /dev/null +++ b/hack/quickstart/nginx-proxy.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-proxy + namespace: kube-system + labels: + k8s-app: kube-nginx +spec: + hostNetwork: true + containers: + - name: nginx-proxy + image: nginx:1.11.4-alpine + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 50m + memory: 32M + requests: + cpu: 25m + memory: 32M + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/nginx + name: etc-nginx + readOnly: true + - name: upstream-sync + image: alpine:3.6 + command: ["/bin/sh", "-c", "eval \"$SCRIPT\""] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + env: + - name: SCRIPT + value: | + set -o nounset -o errexit + trap "kill 0" SIGINT SIGTERM + apk add --update curl + curl -sLo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl + chmod +x /usr/local/bin/kubectl + cd /etc/nginx + while true; do + kubectl --kubeconfig="/etc/kubernetes/kubeconfig" get node -lnode-role.kubernetes.io/master -o go-template --template '{{range .items}}{{"server "}}{{.metadata.name}}{{" backup;\n"}}{{end}}' > upstream.conf.tmp + if cmp -s upstream.conf.tmp upstream.conf; then + mv upstream.conf.tmp upstream.conf + kubectl --kubeconfig="/etc/kubernetes/kubeconfig" -n kube-system exec nginx-proxy-${NODE_NAME} -- nginx -s reload + fi + sleep 600 & + wait + done + volumeMounts: + - mountPath: /etc/nginx + name: etc-nginx + - mountPath: /etc/kubernetes + name: etc-kubernetes + readOnly: true + volumes: + - name: etc-nginx + hostPath: + path: /etc/nginx + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes diff --git a/hack/quickstart/nginx.conf b/hack/quickstart/nginx.conf new file mode 100644 index 000000000..ef06947e9 --- /dev/null +++ b/hack/quickstart/nginx.conf @@ -0,0 +1,22 @@ +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + least_conn; + server 10.3.0.1:443; + include /etc/nginx/upstream.conf; + } + server { + listen 127.0.0.1:6443; + proxy_pass kube_apiserver; + proxy_timeout 10m; + proxy_connect_timeout 1s; + } +}