corekube-onmetal.yaml

 heat_template_version: 2013-05-23

 description: >
   Deploy Kubernetes on a CoreOS cluster that operates on
   Rackspace's Public Cloud (OnMetal Server)
   Maintainer - John Yi <john.yi@rackspace.com> | Github: @jyidiego

 parameter_groups:
 - label: Discovery Server
   parameters:
   - flavor_discovery
   - coreos_image_discovery
   - discovery_net_interface

 - label: Overlord Server
   parameters:
   - flavor_overlord
   - coreos_image_overlord

 - label: Master Server
   parameters:
   - flavor_master
   - coreos_image_master
   - master_net_interface

 - label: Minion Servers
   parameters:
   - flavor_minions
   - coreos_image_minions
   - minion_net_interface

 - label: Network and Access Configuration 
   parameters:
   - flannel_url
   - cluster_subnet
   - keyname

 parameters:
   kubernetes_minion_count:
     description: Number of CoreOS machines to deploy as Kubernetes Minion
     type: number
     default: 3
     constraints:
     - range:
         min: 1
         max: 12
       description: Must be between 2 and 12 servers.
   keyname:
     type: string
     description: Name of keypair to be used for compute instance
   flavor_minions:
     type: string
     default: OnMetal Compute v1
     constraints:
     - allowed_values:
       - OnMetal Compute v1
       - OnMetal IO v1
       - OnMetal Memory v1 
       description: |
         Must be a valid Rackspace OnMetal flavor for the region you have
         selected to deploy into.
   flavor_discovery:
     type: string
     default: 4 GB General Purpose v1
     constraints:
     - allowed_values:
       - 4 GB General Purpose v1
       - 8 GB General Purpose v1
       description: |
         Must be a valid Rackspace Cloud Server flavor for the region you have
         selected to deploy into.
   flavor_overlord:
     type: string
     default: 4 GB General Purpose v1
     constraints:
     - allowed_values:
       - 4 GB General Purpose v1
       - 8 GB General Purpose v1
       description: |
         Must be a valid Rackspace Cloud Server flavor for the region you have
         selected to deploy into.
   flavor_master:
     type: string
     default: 4 GB General Purpose v1
     constraints:
     - allowed_values:
       - 4 GB General Purpose v1
       - 8 GB General Purpose v1
       description: |
         Must be a valid Rackspace Cloud Server flavor for the region you have
         selected to deploy into.
   coreos_image_minions:
     type: string
     description: Rackspace OnMetal CoreOS Stable (633.1.0) UUID
     default: "84997bd3-7cdf-4405-8075-137d631f24b4"
   coreos_image_discovery:
     type: string
     description: Rackspace Cloud Servers CoreOS Stable (633.1.0) UUID
     default: "a41d8b7c-d41b-4369-a180-0734f3b1cd8c"
   coreos_image_overlord:
     type: string
     description: Rackspace Cloud Servers CoreOS Stable (633.1.0) UUID
     default: "a41d8b7c-d41b-4369-a180-0734f3b1cd8c"
   coreos_image_master:
     type: string
     description: Rackspace Cloud Servers CoreOS Stable (633.1.0) UUID
     default: "a41d8b7c-d41b-4369-a180-0734f3b1cd8c"
   git_command:
     type: string
     description: Git repo checkout command
     default: "/usr/bin/git clone https://github.com/metral/overlord ; /usr/bin/git -C overlord checkout -qf 96574056d0cc7ad8e52228df76374cb3537af8ba"
   flannel_url:
     type: string
     description: Flannel (0.4.0) Binary URL
     default: "http://a26736f09be9784cb06f-8d30bd32035bc1ac3e9215d2d6eedfc5.r53.cf1.rackcdn.com/flanneld"
   discovery_net_interface:
     type: string
     description: Host network interface for which the Discovery node will operate over
     default: "eth1"
   overlord_net_interface:
     type: string
     description: Host network interface for which the Discovery node will operate over
     default: "eth1"
   master_net_interface:
     type: string
     description: Host network interface for which the Discovery node will operate over
     default: "eth1"
   minion_net_interface:
     type: string
     description: Host network interface for which Kubernetes' overlay will operate over
     default: "bond0.401"
   cluster_subnet:
     type: string
     description: cluster subnet that containers will operate over
     default: "10.244.0.0/15"

 resources:
   coreos_cluster_uuid:
     type: OS::Heat::RandomString

   discovery:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image_discovery }
       flavor: { get_param: flavor_discovery }
       name: "discovery"
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_interface_ip.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Get's the IP of the interface that discovery will be
                   # accessible over
                   DISCOVERY_IF=%discovery_net_interface%
                   /usr/bin/ip -4 addr show $DISCOVERY_IF | /usr/bin/awk '/inet/ {print $2}' | /usr/bin/cut -d/ -f1 > /run/IP
                   /usr/bin/sed -i 's/^/IP=/' /run/IP
             coreos:
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: private-discovery-setup.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_interface_ip.sh
                 - name: private-discovery.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target private-discovery-setup.service
                     Requires=network-online.target private-discovery-setup.service

                     [Service]
                     EnvironmentFile=/run/IP
                     RestartSec=5s
                     ExecStartPre=/usr/bin/docker pull quay.io/coreos/etcd:v2.0.9
                     ExecStart=/usr/bin/docker run -d --name discovery \
                       -p 2379:2379 -p 2380:2380 \
                       -v /usr/share/ca-certificates/:/etc/ssl/certs \
                       --net host quay.io/coreos/etcd:v2.0.9 -name discovery \
                       -initial-advertise-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -listen-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -initial-cluster discovery=http://${IP}:2380,discovery=http://${IP}:7001 \
                       -advertise-client-urls http://${IP}:2379,http://${IP}:4001 \
                       -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001
           params:
             "%discovery_net_interface%": { get_param: discovery_net_interface }

   overlord:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image_overlord }
       flavor: { get_param: flavor_overlord }
       name: "overlord"
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             coreos:
               etcd:
                 name: overlord
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: etcd.service
                   command: start
                 - name: fleet.socket
                   command: start
                   content: |
                     [Socket]
                     # Talk to the API over a Unix domain socket (default)
                     ListenStream=/var/run/fleet.sock

                     # Talk to the API over an exposed port
                     ListenStream=10001
                     Service=fleet-local.service

                     [Install]
                     WantedBy=sockets.target
                 - name: fleet-local.service
                   command: start
                   content: |
                     # fleet-local is kicked off by fleet.socket after API port
                     # is opened
                     [Unit]
                     Description=fleet-local
                     Wants=etcd.service
                     After=etcd.service

                     [Service]
                     Environment=FLEET_PUBLIC_IP=$private_ipv4
                     Environment=FLEET_METADATA=kubernetes_role=overlord
                     ExecStart=/usr/bin/fleet
                     Restart=always
                     RestartSec=10s
                 - name: overlord.service
                   command: start
                   content: |
                     # Overlord / logic layer service to deploy kubernetes to
                     # the cluster
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     WorkingDirectory=/root
                     Environment="DIR=overlord"
                     ExecStartPre=/usr/bin/rm -rf $DIR
                     ExecStartPre=%git_command%
                     ExecStart=/usr/bin/bash ${DIR}/build_run.sh
           params:
             "%discovery%": { get_attr: [discovery, networks, private, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%git_command%": { get_param: git_command }

   kubernetes_master:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image_master }
       flavor: { get_param: flavor_master}
       name: "kubernetes_master"
       networks:
       - uuid: "00000000-0000-0000-0000-000000000000"
       - uuid: "11111111-1111-1111-1111-111111111111"
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_ip_port.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Sets up environment file with the discovery node's IP &
                   # port so # that in Overlode's template 
                   # master-apiserver@.service it can be passed 
                   # in as an argument
                   /usr/bin/cat /run/systemd/system/etcd.service.d/20-cloudinit.conf | /usr/bin/grep -i discovery | /usr/bin/cut -f3 -d"=" | /usr/bin/awk -F '/v' '{print $1}' > /run/discovery_ip_port
                   /usr/bin/sed -i 's/^/DISCOVERY_IP_PORT=/' /run/discovery_ip_port
             coreos:
               etcd:
                 name: kubernetes_master
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
               fleet:
                 public-ip: $private_ipv4
                 metadata: kubernetes_role=master
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: etcd.service
                   command: start
                 - name: create-etcd-env.service
                   command: start
                   content: |
                     [Unit]
                     Description=creates etcd environment
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     Type=oneshot
                     ExecStart=/bin/sh -c "sed -i \"s/=:/=`ifconfig %kubernetes_net_interface% | grep 'inet ' | awk '{print $2}'`:/\" /run/systemd/system/etcd.service.d/20-cloudinit.conf && systemctl daemon-reload"
                 - name: fleet.service
                   command: start
                 - name: get_discovery_ip_port.service
                   runtime: true
                   command: start
                   content: |
                     # Runs get_discovery_ip_port.sh to have discovery IP &
                     # port ready for consumption by overlord when creating
                     # the Kubernetes' master-api@.service template
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_ip_port.sh
                 - name: flannel-install.service
                   command: start
                   content: |
                     # Installs flannel
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     Type=oneshot
                     RemainAfterExit=yes
                     ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                     ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                 - name: flannel.service
                   command: start
                   content: |
                     # Configures & starts flannel
                     [Unit]
                     After=network-online.target etcd.service flannel-install.service
                     Requires=network-online.target etcd.service flannel-install.service

                     [Service]
                     ExecStartPre=/usr/bin/etcdctl mk /coreos.com/network/config '{"Network":"%cluster_subnet%", "Backend": {"Type": "udp"}}'
                     ExecStart=/opt/bin/flanneld -iface=%kubernetes_net_interface%
                     Restart=always
                     RestartSec=5s
                 - name: flannel-env.path
                   command: start
                   content: |
                     # Ensures flannel env vars are set to use with Docker
                     [Path]
                     PathExists=/run/flannel/subnet.env
                     Unit=docker.service
                 - name: docker.service
                   command: start
                   content: |
                     # Starts new docker server that uses flannel 
                     [Unit]
                     After=flannel-env.path network-online.target flannel.service
                     Requires=flannel-env.path network-online.target flannel.service
                     Description=Docker Application Container Engine

                     [Service]
                     EnvironmentFile=/run/flannel/subnet.env
                     ExecStartPre=/bin/mount --make-rprivate /
                     ExecStartPre=/usr/bin/systemctl kill docker.service
                     ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                     [Install]
                     WantedBy=multi-user.target
           params:
             "%discovery%": { get_attr: [discovery, networks, private, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%flannel_url%": { get_param: flannel_url }
             "%kubernetes_net_interface%": { get_param: master_net_interface }
             "%cluster_subnet%": { get_param: cluster_subnet }

   kubernetes_minions:
     type: "OS::Heat::ResourceGroup"
     properties:
       count: { get_param: kubernetes_minion_count }
       resource_def:
         type: OS::Nova::Server
         properties:
           key_name: { get_param: keyname }
           image: { get_param: coreos_image_minions }
           flavor: { get_param: flavor_minions}
           name: kubernetes_minion_%index%
           networks:
           - uuid: "00000000-0000-0000-0000-000000000000"
           - uuid: "11111111-1111-1111-1111-111111111111"
           user_data_format: RAW
           config_drive: "true"
           user_data:
             str_replace:
               template: |
                 #cloud-config
                 coreos:
                   etcd:
                     name: kubernetes_minion_%index%
                     addr: $private_ipv4:4001
                     peer-addr: $private_ipv4:7001
                     discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                   fleet:
                     public-ip: $private_ipv4
                     metadata: kubernetes_role=minion
                   update:
                     group: stable
                     reboot-strategy: off
                   units:
                     - name: etcd.service
                       command: start
                     - name: create-etcd-env.service
                       command: start
                       content: |
                         [Unit]
                         Description=creates etcd environment
                         After=network-online.target
                         Requires=network-online.target

                         [Service]
                         Type=oneshot
                         ExecStart=/bin/sh -c "sed -i \"s/=:/=`ifconfig %kubernetes_net_interface% | grep 'inet ' | awk '{print $2}'`:/\" /run/systemd/system/etcd.service.d/20-cloudinit.conf && systemctl daemon-reload"
                     - name: fleet.service
                       command: start
                     - name: flannel-install.service
                       command: start
                       content: |
                         # Installs flannel
                         [Unit]
                         After=network-online.target
                         Requires=network-online.target

                         [Service]
                         Type=oneshot
                         RemainAfterExit=yes
                         ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                         ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                     - name: flannel.service
                       command: start
                       content: |
                         # Configures & starts flannel
                         [Unit]
                         After=etcd.service flannel-install.service
                         Requires=etcd.service flannel-install.service

                         [Service]
                         ExecStart=/opt/bin/flanneld -iface=%kubernetes_net_interface%
                         Restart=always
                         RestartSec=5s
                     - name: flannel-env.path
                       command: start
                       content: |
                         # Ensures flannel env vars are set to use with Docker
                         [Path]
                         PathExists=/run/flannel/subnet.env
                         Unit=docker.service
                     - name: docker.service
                       command: start
                       content: |
                         # Starts new docker server that uses flannel 
                         [Unit]
                         After=flannel-env.path network-online.target flannel.service
                         Requires=flannel-env.path network-online.target flannel.service
                         Description=Docker Application Container Engine

                         [Service]
                         EnvironmentFile=/run/flannel/subnet.env
                         ExecStartPre=/bin/mount --make-rprivate /
                         ExecStartPre=/usr/bin/systemctl kill docker.service
                         ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                         [Install]
                         WantedBy=multi-user.target
               params:
                 "%discovery%": { get_attr: [discovery, networks, private, 0] }
                 "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
                 "%flannel_url%": { get_param: flannel_url }
                 "%kubernetes_net_interface%": { get_param: minion_net_interface }

 outputs:
   discovery_ip:
     value: { get_attr: [ discovery, accessIPv4 ] }
     description: The IP of the Discovery
   overlord_ip:
     value: { get_attr: [ overlord, accessIPv4 ] }
     description: The IP of the Overlord
   master_ip:
     value: { get_attr: [ kubernetes_master, accessIPv4 ] }
     description: The IP of the Kubernetes Master
   minion_ips:
     value: { get_attr: [ kubernetes_minions, accessIPv4 ] }
     description: The IP of the Kubernetes Minions