Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

hack/quickstart: Add automatic self-hosted multi-master loadbalancer #684

Closed
wants to merge 2 commits into from
Closed

hack/quickstart: Add automatic self-hosted multi-master loadbalancer #684

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
116593d
hack/quickstart: Add automatic self-hosted multi-master loadbalancer
klausenbusk Aug 13, 2017
4a55a33
foo
klausenbusk Aug 28, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions hack/quickstart/init-master.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ REMOTE_USER=${REMOTE_USER:-core}
CLUSTER_DIR=${CLUSTER_DIR:-cluster}
IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
SSH_OPTS=${SSH_OPTS:-}
MULTI_MASTER=${MULTI_MASTER:-false}
SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY:-false}
CLOUD_PROVIDER=${CLOUD_PROVIDER:-}
Expand Down Expand Up @@ -68,14 +69,23 @@ function init_master_node() {

# Render cluster assets
/home/${REMOTE_USER}/bootkube render --asset-dir=/home/${REMOTE_USER}/assets ${etcd_render_flags} ${cnp_render_flags} \
--api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
--api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443,https://127.0.0.1:6443

# Move the local kubeconfig into expected location
chown -R ${REMOTE_USER}:${REMOTE_USER} /home/${REMOTE_USER}/assets
mkdir -p /etc/kubernetes
cp /home/${REMOTE_USER}/assets/auth/kubeconfig /etc/kubernetes/
cp /home/${REMOTE_USER}/assets/tls/ca.crt /etc/kubernetes/ca.crt

if [ "$MULTI_MASTER" = true ]; then
sed -r -e "s/(server: ).*/\1https:\/\/127.0.0.1:6443/" -i /etc/kubernetes/kubeconfig
mkdir -p /etc/nginx
cp nginx.conf /etc/nginx
echo "server 127.0.0.1:443 backup;" > /etc/nginx/upstream.conf
mkdir -p /etc/kubernetes/manifests
cp nginx-proxy.yaml /etc/kubernetes/manifests
fi

# Start etcd.
if [ "$SELF_HOST_ETCD" = false ] ; then
configure_etcd
Expand Down Expand Up @@ -119,16 +129,22 @@ if [ "${REMOTE_HOST}" != "local" ]; then
exit 1
fi
fi

if [ "$MULTI_MASTER" = true ]; then
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx.conf
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx-proxy.yaml ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx-proxy.yaml
fi

# Copy self to remote host so script can be executed in "local" mode
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/init-master.sh
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} SELF_HOST_ETCD=${SELF_HOST_ETCD} CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY} /home/${REMOTE_USER}/init-master.sh local"
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} MULTI_MASTER=${MULTI_MASTER} SELF_HOST_ETCD=${SELF_HOST_ETCD} CALICO_NETWORK_POLICY=${CALICO_NETWORK_POLICY} /home/${REMOTE_USER}/init-master.sh local"

# Copy assets from remote host to a local directory. These can be used to launch additional nodes & contain TLS assets
mkdir ${CLUSTER_DIR}
scp -q -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} -r ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/assets/* ${CLUSTER_DIR}

# Cleanup
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm -rf /home/${REMOTE_USER}/{assets,init-master.sh,bootkube}"
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm -rf /home/${REMOTE_USER}/{assets,init-master.sh,bootkube,nginx.conf,nginx-proxy.yaml}"

echo "Cluster assets copied to ${CLUSTER_DIR}"
echo
Expand Down
20 changes: 18 additions & 2 deletions hack/quickstart/init-node.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ REMOTE_USER=${REMOTE_USER:-core}
IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
SSH_OPTS=${SSH_OPTS:-}
TAG_MASTER=${TAG_MASTER:-false}
MULTI_MASTER=${MULTI_MASTER:-false}
CLOUD_PROVIDER=${CLOUD_PROVIDER:-}

function usage() {
Expand All @@ -22,6 +23,14 @@ function init_worker_node() {
# Setup kubeconfig
mkdir -p /etc/kubernetes
cp ${KUBECONFIG} /etc/kubernetes/kubeconfig
if [ "$MULTI_MASTER" = true ]; then
sed -r -e "s/(server: ).*/\1https:\/\/127.0.0.1:6443/" -i /etc/kubernetes/kubeconfig
mkdir -p /etc/nginx
cp nginx.conf /etc/nginx
cp upstream.conf /etc/nginx
mkdir -p /etc/kubernetes/manifests
cp nginx-proxy.yaml /etc/kubernetes/manifests
fi
# Pulled out of the kubeconfig. Other installations should place the root
# CA here manually.
grep 'certificate-authority-data' ${KUBECONFIG} | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt
Expand Down Expand Up @@ -51,12 +60,19 @@ if [ "${REMOTE_HOST}" != "local" ]; then
fi
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${KUBECONFIG} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/kubeconfig

if [ "$MULTI_MASTER" = true ]; then
kubectl --kubeconfig=${KUBECONFIG} get node -lnode-role.kubernetes.io/master -o go-template --template '{{range .items}}{{"server "}}{{.metadata.name}}{{" backup;\n"}}{{end}}' > upstream.conf
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx.conf
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} upstream.conf ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/upstream.conf
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} nginx-proxy.yaml ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/nginx-proxy.yaml
fi

# Copy self to remote host so script can be executed in "local" mode
scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} ${REMOTE_USER}@${REMOTE_HOST}:/home/${REMOTE_USER}/init-node.sh
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} CLOUD_PROVIDER=${CLOUD_PROVIDER} /home/${REMOTE_USER}/init-node.sh local /home/${REMOTE_USER}/kubeconfig"
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "sudo REMOTE_USER=${REMOTE_USER} MULTI_MASTER=${MULTI_MASTER} CLOUD_PROVIDER=${CLOUD_PROVIDER} /home/${REMOTE_USER}/init-node.sh local /home/${REMOTE_USER}/kubeconfig"

# Cleanup
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm /home/${REMOTE_USER}/{init-node.sh,kubeconfig}"
ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} ${REMOTE_USER}@${REMOTE_HOST} "rm /home/${REMOTE_USER}/{init-node.sh,kubeconfig,nginx-proxy.yaml,nginx.conf}"

echo
echo "Node bootstrap complete. It may take a few minutes for the node to become ready. Access your kubernetes cluster using:"
Expand Down
66 changes: 66 additions & 0 deletions hack/quickstart/nginx-proxy.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
apiVersion: v1
kind: Pod
metadata:
name: nginx-proxy
namespace: kube-system
labels:
k8s-app: kube-nginx
spec:
hostNetwork: true
containers:
- name: nginx-proxy
image: nginx:1.11.4-alpine
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 50m
memory: 32M
requests:
cpu: 25m
memory: 32M
securityContext:
privileged: true
volumeMounts:
- mountPath: /etc/nginx
name: etc-nginx
readOnly: true
- name: upstream-sync
image: alpine:3.6
command: ["/bin/sh", "-c", "eval \"$SCRIPT\""]
env:
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
env:
- name: SCRIPT
value: |
set -o nounset -o errexit
trap "kill 0" SIGINT SIGTERM
apk add --update curl
curl -sLo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod +x /usr/local/bin/kubectl
cd /etc/nginx
while true; do
kubectl --kubeconfig="/etc/kubernetes/kubeconfig" get node -lnode-role.kubernetes.io/master -o go-template --template '{{range .items}}{{"server "}}{{.metadata.name}}{{" backup;\n"}}{{end}}' > upstream.conf.tmp
if cmp -s upstream.conf.tmp upstream.conf; then
mv upstream.conf.tmp upstream.conf
kubectl --kubeconfig="/etc/kubernetes/kubeconfig" -n kube-system exec nginx-proxy-${NODE_NAME} -- nginx -s reload
fi
sleep 600 &
wait
done
volumeMounts:
- mountPath: /etc/nginx
name: etc-nginx
- mountPath: /etc/kubernetes
name: etc-kubernetes
readOnly: true
volumes:
- name: etc-nginx
hostPath:
path: /etc/nginx
- name: etc-kubernetes
hostPath:
path: /etc/kubernetes
22 changes: 22 additions & 0 deletions hack/quickstart/nginx.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
error_log stderr notice;

worker_processes auto;
events {
multi_accept on;
use epoll;
worker_connections 1024;
}

stream {
upstream kube_apiserver {
least_conn;
server 10.3.0.1:443;
include /etc/nginx/upstream.conf;
}
server {
listen 127.0.0.1:6443;
proxy_pass kube_apiserver;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
}