Skip to content
This repository has been archived by the owner on May 22, 2020. It is now read-only.

min-turnup: azure #106

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions min-turnup/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
FROM docker.io/buildpack-deps:xenial

RUN bash -c "\
apt-get update && apt-get -y upgrade && \
apt-get install -y make unzip jq && \
rm -rf /var/lib/apt/lists/*"

ENV TERRAFORM_VERSION 0.7.0-rc2
RUN bash -c "\
wget -q -O /tmp/terraform.zip \"https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip\" && \
unzip /tmp/terraform.zip -d /usr/local/bin/ && \
rm /tmp/terraform.zip"

################################################################################################
# Azure - Required dependencies
################################################################################################
RUN bash -c "\
apt-get update && apt-get -y upgrade && \
apt-get install -y nodejs npm && \
rm -rf /var/lib/apt/lists/*"
RUN bash -c "npm install -g azure-cli"

################################################################################################

################################################################################################
# Jsonnet
################################################################################################

RUN bash -c "\
cd /tmp; \
git clone https://github.com/google/jsonnet; \
cd jsonnet; \
make; \
cp jsonnet /usr/bin/jsonnet;"

WORKDIR /opt/kube-min-turnup

ADD . /opt/kube-min-turnup/

CMD make
20 changes: 18 additions & 2 deletions min-turnup/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@

SHELL=/bin/bash
.SHELLFLAGS="-O extglob -o errexit -o pipefail -o nounset -c"

# TODO: things break with this uncommented
#.SHELLFLAGS="-O extglob -o errexit -o pipefail -o nounset -c"

.PHONY: config echo-config

Expand All @@ -17,6 +19,16 @@ endif
CONF_TOOL_VERSION = 4.6
KCONFIG_FILES = $(shell find . -name 'Kconfig')


docker-build:
docker build -t kube-min-turnup .

docker-run: docker-build
docker run -it --net=host kube-min-turnup /bin/bash

docker-dev: docker-build
docker run -it --net=host -v `pwd`:/opt/kube-min-turnup kube-min-turnup /bin/bash

default:
$(MAKE) config

Expand All @@ -40,11 +52,15 @@ menuconfig: .tmp/mconf
$(MAKE) config

.config.json: .config
util/conig_to_json $< > $@
util/config_to_json $< > $@

echo-config: .config.json
cat $<

set-kubeconfig:
kubectl config set-cluster $(CLUSTER_NAME) --server https://$(MASTER_IP):443 --certificate-authority=phase1b/crypto/ca.pem --embed-certs=true
kubectl config set-credentials $(CLUSTER_NAME) --server https://$(MASTER_IP):443 --client-certificate=./phase1b/crypto/kubelet.pem --client-key=./phase1b/crypto/kubelet-key.pem --embed-certs=true
kubectl config set-context $(CLUSTER_NAME) --cluster $(CLUSTER_NAME) --user $(CLUSTER_NAME)

clean:
rm -rf .tmp
Expand Down
31 changes: 0 additions & 31 deletions min-turnup/default.config

This file was deleted.

10 changes: 7 additions & 3 deletions min-turnup/phase1/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,24 @@ config phase1.num_nodes

config phase1.instance_prefix
string "instance prefix"
default "kuberentes"
default "k0"
help
A prefix to append to all instance names.

menuconfig phase1.cloud_provider
string "cloud proivder"
string "cloud provider"
default "gce"
help
The cloud provider you would like to deploy to.

Valid options are (gce).
Valid options are (gce, azure).

if phase1.cloud_provider = "gce"
source "phase1/gce/Kconfig"
endif

if phase1.cloud_provider = "azure"
source "phase1/azure/Kconfig"
endif

endmenu
3 changes: 3 additions & 0 deletions min-turnup/phase1/azure/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
terraform.tfstate
account.json
terraform.tfstate.backup
76 changes: 76 additions & 0 deletions min-turnup/phase1/azure/Kconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
menu "Azure configuration"

config phase1.azure.image_publisher
string "Base Virtual Machine OS Image"
default "canonical"
help
The publisher of the base image used for the VirtualMachines.
config phase1.azure.image_offer
string "Base Virtual Machine OS Image"
default "ubuntuserver"
help
The offer of the base image used for the VirtualMachines.
config phase1.azure.image_sku
string "Base Virtual Machine OS Image"
default "16.04.0-LTS"
help
The sku of the base image used for the VirtualMachines.
config phase1.azure.image_version
string "Base Virtual Machine OS Image"
default "latest"
help
The version of the base image used for the VirtualMachines.

config phase1.azure.master_vm_size
string "Virtual Machine Size (Master)"
default "Standard_D1_v2"
help
The size of VirtualMachine to deploy.

config phase1.azure.node_vm_size
string "Virtual Machine Size (Node)"
default "Standard_D1_v2"
help
The size of VirtualMachine to deploy.

config phase1.azure.master_private_ip
string "Private IP address of Master"
default "10.0.1.4"
help
The private ip address of master

config phase1.azure.location
string "Resource Location"
default "westus"
help
The Azure location to use.

config phase1.azure.admin_username
string "Virtual Machine Admin Username"
default "kube"

config phase1.azure.admin_password
string "Virtual Machine Admin Password"
default "AzureKubernet3s!"

config phase1.azure.tenant_id
string "ActiveDirectory ServicePrincipal ClientSecret"
help
The ClientSecret of the Service Account to be used by the cluster components.

config phase1.azure.subscription_id
string "ActiveDirectory ServicePrincipal ClientSecret"
help
The ClientSecret of the Service Account to be used by the cluster components.

config phase1.azure.client_id
string "ActiveDirectory ServicePrincipal ClientID"
help
The ClientID of the Service Account to be used by the cluster components.

config phase1.azure.client_secret
string "ActiveDirectory ServicePrincipal ClientSecret"
help
The ClientSecret of the Service Account to be used by the cluster components.

endmenu
4 changes: 4 additions & 0 deletions min-turnup/phase1/azure/all.jsonnet
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
local cfg = import "../../.config.json";
{
"azure.tf": (import "lib/azure.jsonnet")(cfg),
}
11 changes: 11 additions & 0 deletions min-turnup/phase1/azure/azure.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"tenantId": "${tenantId}",
"subscriptionId": "${subscriptionId}",
"adClientId": "${adClientId}",
"adClientSecret": "${adClientSecret}",
"resourceGroup": "${resourceGroup}",
"location": "${location}",
"subnetName": "${subnetName}",
"securityGroupName": "${securityGroupName}",
"vnetName": "${vnetName}"
}
68 changes: 68 additions & 0 deletions min-turnup/phase1/azure/configure-vm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#! /bin/bash

set -x
set -o errexit
set -o pipefail
set -o nounset

mkdir -p /etc/systemd/system/docker.service.d/
cat <<EOF > /etc/systemd/system/docker.service.d/clear_mount_propagtion_flags.conf
[Service]
MountFlags=shared
EOF
cat <<EOF > /etc/systemd/system/docker.service.d/overlay.conf
[Service]
ExecStart=
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am no systemd expert. Why is this line necessary?

Copy link

@errordeveloper errordeveloper Jun 14, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well spotted ;) This resets the previous value. Well, it's a bit of a special case for ExecStart and certain other attributes, but that's what you have to do and it complains otherwise.

ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay
EOF

curl -sSL https://get.docker.com/ | sh

apt-get install -y jq

systemctl start docker || true

ROLE="node"
if [[ $(hostname) = *master* ]]; then
ROLE="master"
fi

azure_file="/etc/kubernetes/azure.json"
config_file="/etc/kubernetes/k8s_config.json"

mkdir -p /etc/kubernetes

# the following values are populated by terraform
echo -n "${azure_json}" | base64 -d > "$azure_file"
echo -n "${k8s_config}" | base64 -d > "$config_file"
echo -n "${kubelet_tar}" | base64 -d > "/etc/kubernetes/kubelet.tar"
echo -n "${root_tar}" | base64 -d > "/etc/kubernetes/root.tar"
echo -n "${apiserver_tar}" | base64 -d > "/etc/kubernetes/apiserver.tar"

MASTER_IP="$(cat "$config_file" | jq -r '.phase1.azure.master_private_ip')"

jq ". + {\"role\": \"$ROLE\", \"master_ip\": \"$MASTER_IP\"}" "$config_file" > /etc/kubernetes/k8s_config.new; cp /etc/kubernetes/k8s_config.new "$config_file"

mkdir -p /srv/kubernetes
for bundle in root kubelet apiserver; do
cat "/etc/kubernetes/$bundle.tar" | sudo tar xv -C /srv/kubernetes
done;

installer_container_spec="$(cat "$config_file" | jq -r '.phase2.installer_container_spec')"

cat << EOF > /etc/kubernetes/install.sh
#!/bin/bash
systemctl stop docker
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this script could use a #!/bin/bash. why do you need to restart docker?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RE: this and the question above... I had done some hacky things in configure-vm.sh. In Azure, the configure-vm.sh script didn't seem to be available in plaintext (there might be a base64 copy somewhere). So what I was doing was writing the install.sh script out to disk.

So the configure-vm.sh dropped the install.sh script for future runs, as I was iterating on the Ansible container and then invoked it at the end of configure-vm.sh for the initial deployment.

The thinking here was that someone would want to upgrade their cluster. They've updated the k8s_config.json to reference the newer Kubernetes version, now they need to re-run the playbook container. With my setup, they just call install.sh again. Kubelet gets torn down as part of rebooting Docker. Playbook runs, new kubelet boots up. So install.sh is what I was treating as needing to be idempotent.

(and yes, it should have a shebang, if we want to keep this model. I don't know if configure-vm.sh is meant to be re-run on GCE?)

systemctl start docker
docker pull "$installer_container_spec"
docker run \
--net=host \
-v /:/host_root \
-v /etc/kubernetes/k8s_config.json:/opt/playbooks/config.json:ro \
"$installer_container_spec" \
/opt/do_role.sh "$ROLE"
EOF

chmod +x /etc/kubernetes/install.sh
/etc/kubernetes/install.sh

8 changes: 8 additions & 0 deletions min-turnup/phase1/azure/gen
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#! /bin/bash

set -x
set -o errexit
set -o pipefail
set -o nounset

jsonnet --multi out/ all.jsonnet
Loading