Add docs for atlas, add push config for do/googlecompute, refactor na…

…mes of images
Capgemini · Apr 30, 2015 · 163c735 · sheerun · Jan 5, 2016 · 163c735
1 parent cff6c80
commit 163c735
Show file tree

Hide file tree

Showing 15 changed files with 128 additions and 56 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -20,7 +20,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     raise "vagrant-hosts plugin not installed"
   end
 
-  config.vm.box = "capgemini/mesos"
+  config.vm.box = "capgemini/apollo-mesos"
   config.vm.box_version = conf['mesos_version']
 
   ansible_groups = {

diff --git a/bootstrap/aws/config-default.sh b/bootstrap/aws/config-default.sh
@@ -7,6 +7,8 @@ AWS_SSH_KEY=${AWS_SSH_KEY:-$HOME/.ssh/apollo_aws_rsa}
 AWS_SSH_KEY_NAME=${AWS_SSH_KEY_NAME:-apollo}
 ATLAS_TOKEN=${ATLAS_TOKEN:?"Need to set ATLAS_TOKEN non-empty"}
 ATLAS_INFRASTRUCTURE=${ATLAS_INFRASTRUCTURE:-capgemini/apollo}
+ATLAS_ARTIFACT_MASTER=${ATLAS_ARTIFACT_MASTER:-capgemini/apollo-mesos-ubuntu-14.04-amd64}
+ATLAS_ARTIFACT_SLAVE=${ATLAS_ARTIFACT_SLAVE:-capgemini/apollo-mesos-ubuntu-14.04-amd64}
 ZONE=${APOLLO_AWS_ZONE:-eu-west-1b}
 MASTER_SIZE=${MASTER_SIZE:-m1.medium}
 SLAVE_SIZE=${SLAVE_SIZE:-m1.medium}

diff --git a/bootstrap/aws/util.sh b/bootstrap/aws/util.sh
@@ -96,6 +96,8 @@ terraform_apply() {
       -var "atlas_infrastructure=${ATLAS_INFRASTRUCTURE}" \
       -var "instance_type.master=${MASTER_SIZE}" \
       -var "instance_type.slave=${SLAVE_SIZE}" \
+      -var "atlas_artifact.master=${ATLAS_ARTIFACT_MASTER}" \
+      -var "atlas_artifact.slave=${ATLAS_ARTIFACT_SLAVE}" \
       -var "slaves=${NUM_SLAVES}" \
       -var "subnet_availability_zone=${ZONE}" \
       -var "region=${AWS_REGION}"

diff --git a/bootstrap/digitalocean/config-default.sh b/bootstrap/digitalocean/config-default.sh
@@ -1,23 +1,4 @@
 #!/bin/bash
-get_lastest_apollo_image_id() {
-  api_call() {
-    curl -s -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer ${DIGITALOCEAN_API_TOKEN}" "https://api.digitalocean.com/v2/images?private=true"
-  }
-  return_id_name_table() {
-    api_call | awk -F"[,:}]" '{for(i=1;i<=NF;i++){if($i~/'id'\042/){print $(i+1)"\t"$(i+3)}}}' | tr -d '"' | sed -n p
-  }
-  return_name_with_apollo_rows() {
-    return_id_name_table | grep apollo
-  }
-  return_ids_column() {
-    return_name_with_apollo_rows | awk -F"\t" '{print $1}'
-  }
-  return_lastest_row() {
-    return_ids_column | awk 'END{print}'
-  }
-  echo $(return_lastest_row)
-}
-
 DIGITALOCEAN_REGION=${DIGITALOCEAN_REGION:-lon1}
 DIGITALOCEAN_SSH_PUB_KEY=${DIGITALOCEAN_SSH_KEY:?"Need to set DIGITALOCEAN_SSH_KEY non-empty"}
 DIGITALOCEAN_API_TOKEN=${DIGITALOCEAN_API_TOKEN:?"Need to set DIGITALOCEAN_API_TOKEN non-empty"}
@@ -26,11 +7,11 @@ DO_API_KEY=${DO_API_KEY:?"Need to set DO_API_KEY non-empty"}
 
 ATLAS_TOKEN=${ATLAS_TOKEN:?"Need to set ATLAS_TOKEN non-empty"}
 ATLAS_INFRASTRUCTURE=${ATLAS_INFRASTRUCTURE:-capgemini/apollo}
+ATLAS_ARTIFACT_MASTER=${ATLAS_ARTIFACT_MASTER:-capgemini/apollo-mesos-ubuntu-14.04-amd64}
+ATLAS_ARTIFACT_SLAVE=${ATLAS_ARTIFACT_SLAVE:-capgemini/apollo-mesos-ubuntu-14.04-amd64}
 MASTER_SIZE=${MASTER_SIZE:-512mb}
 SLAVE_SIZE=${SLAVE_SIZE:-512mb}
 NUM_SLAVES=${NUM_SLAVES:-1}
-MASTER_IMAGE=${MASTER_IMAGE:-$(get_lastest_apollo_image_id)}
-SLAVE_IMAGE=${SLAVE_IMAGE:-$(get_lastest_apollo_image_id)}
 
 CONSUL_DC=${CONSUL_DC:-$DIGITALOCEAN_REGION}
 MESOS_CLUSTER_NAME=${MESOS_CLUSTER_NAME:-$DIGITALOCEAN_REGION}
diff --git a/bootstrap/digitalocean/util.sh b/bootstrap/digitalocean/util.sh
@@ -49,8 +49,8 @@ terraform_apply() {
       -var "key_file=${DIGITALOCEAN_SSH_KEY}" \
       -var "instance_size.master=${MASTER_SIZE}" \
       -var "instance_size.slave=${SLAVE_SIZE}" \
-      -var "image.master=${MASTER_IMAGE}" \
-      -var "image.slave=${SLAVE_IMAGE}" \
+      -var "atlas_artifact.master=${ATLAS_ARTIFACT_MASTER}" \
+      -var "atlas_artifact.slave=${ATLAS_ARTIFACT_SLAVE}" \
       -var "slaves=${NUM_SLAVES}" \
       -var "region=${DIGITALOCEAN_REGION}"
   popd

diff --git a/docs/atlas.md b/docs/atlas.md
@@ -0,0 +1,37 @@
+## Atlas
+
+We use [Atlas](https://atlas.hashicorp.com) as a centralised storage point for artifacts across cloud providers (and for the Vagrant box).
+
+Since only AWS supports public images (AMIs) we have made the AWS artifact available publicly.
+
+Digitalocean only support private images and the ability to share images privately. There is some discussion around public images at this thread [https://digitalocean.uservoice.com/forums/136585-digitalocean/suggestions/3249642-share-an-image-w-another-account](https://digitalocean.uservoice.com/forums/136585-digitalocean/suggestions/3249642-share-an-image-w-another-account).
+
+On Google cloud the only public images available are maintained by Google. They follow a similar approach to Digitalocean. You can read about their policy here [https://cloud.google.com/compute/docs/images#public_images](https://cloud.google.com/compute/docs/images#public_images).
+
+### Creating your own artifacts in Atlas
+
+If you want to build your own Atlas artifacts for use on Digitalocean(required) then you will need to build and push the artifact yourself in the Atlas cloud.
+
+This is so the artifact is associated with your Digitalocean / Atlas account.
+
+To build the Digitalocean image in atlas do the following:
+
+1. Edit ```packer/ubuntu-14.04_amd64-droplet.json``` to reference your organisation in
+the push and post-processor config. At the moment there is a bug in packer which does not allow us to do variable interpolation in push config, see [https://github.com/mitchellh/packer/issues/1861](https://github.com/mitchellh/packer/issues/1861)
+
+```
+cd packer/
+packer push -token=$ATLAS_TOKEN -m="Optional message here" ubuntu-14.04_amd64-droplet.json
+```
+
+replacing $ATLAS_TOKEN with your Atlas API token.
+
+This should create a build configuration under -
+
+https://atlas.hashicorp.com/YOUR_OGRANISATION/build-configurations/apollo-mesos-ubuntu-14-04-amd64
+
+The build will probably fail first time, this will be due to Atlas not having your API token. You will need to add the DIGITALOCEAN_API_TOKEN as a variable under https://atlas.hashicorp.com/YOUR_ORGANISATION/build-configurations/apollo-mesos-ubuntu-14-04-amd64/variables.
+
+Once your build is completed there should be a new artifact under https://atlas.hashicorp.com/YOUR_ORGANISATION/artifacts/apollo-mesos-ubuntu-14.04-amd64
+
+The same steps apply if you want to create your own build configuration / artifacts for AWS/Google.
diff --git a/docs/getting-started-guides/digitalocean.md b/docs/getting-started-guides/digitalocean.md
@@ -21,6 +21,18 @@ droplets. Execute ```ssh-keygen -t rsa``` to create a key pair.
 1. ```git clone https://github.com/Capgemini/apollo.git```
 2. ```cd apollo```
 
+#### Build the base image in Atlas
+```
+cd packer
+packer build ubuntu-14.04_amd64-droplet.json
+```
+
+We are using [Atlas](https://atlas.hashicorp.com) to store artifacts (images) for
+builds.
+
+You will need to push and build your Atlas artifact (via Packer in the Atlas cloud) in order to
+provision your nodes via terraform. See the [guide on how to build artifacts in Atlas](../../docs/atlas.md)
+
 #### Set config
 
 Configuration can be set via environment variables. For a full list of available config
@@ -52,6 +64,8 @@ DIGITALOCEAN_SSH_KEY=
 # Atlas variables.
 ATLAS_TOKEN=
 ATLAS_INFRASTRUCTURE=
+ATLAS_ARTIFACT_MASTER=
+ATLAS_ARTIFACT_SLAVE=
 ```
 
 By default Apollo will try to set the master/slave id to the latest version of your Apollo image from your Digitalocean account.
@@ -62,13 +76,6 @@ MASTER_IMAGE=
 SLAVE_IMAGE=
 ```
 
-#### Build the base image
-```
-cd packer
-packer build ubuntu-14.04_amd64-droplet.json
-```
-This will build and upload the image into your Digitalocean account.
-
 #### Turn up the cluster
 ```
 sh bootstrap/apollo-launch.sh

diff --git a/packer/ubuntu-14.04_amd64-amis.json b/packer/ubuntu-14.04_amd64-amis.json
@@ -6,11 +6,11 @@
     "marathon_version": "0.8.1-1.0.171.ubuntu1404",
     "consul_version": "0.5.0",
     "weave_version": "v0.10.0",
-    "build_version": "{{ timestamp }}"
+    "version": "0.22.1"
   },
   "builders": [{
     "type": "amazon-ebs",
-    "ami_name": "mesos-{{user `mesos_version`}}_amd64_{{user `build_version`}}",
+    "ami_name": "apollo-mesos-ubuntu-14.04-amd64",
     "ami_description": "Ubuntu 14.04 LTS, Meosos {{user `mesos_version`}}, Docker, Marathon, Consul {{user `consul_version`}}, Weave {{user `weave_version`}}, and Zookeeper.",
     "access_key": "{{user `aws_access_key`}}",
     "secret_key": "{{user `aws_secret_key`}}",
@@ -60,14 +60,15 @@
     }
   ],
   "push": {
-    "name": "capgemini/mesos-0-22-1-ubuntu-14-04"
+    "name": "capgemini/apollo-mesos-ubuntu-14-04-amd64"
   },
   "post-processors": [{
     "type": "atlas",
-    "artifact": "capgemini/mesos-0.22.1_ubuntu-14.04_amd64",
+    "artifact": "capgemini/apollo-mesos-ubuntu-14.04-amd64",
     "artifact_type": "aws.ami",
     "metadata": {
-      "created_at": "{{timestamp}}"
+      "created_at": "{{timestamp}}",
+      "version": "{{user `version`}}"
     }
   }]
 }
diff --git a/packer/ubuntu-14.04_amd64-droplet.json b/packer/ubuntu-14.04_amd64-droplet.json
@@ -7,11 +7,11 @@
     "marathon_version": "0.8.1-1.0.171.ubuntu1404",
     "consul_version": "0.5.0",
     "weave_version": "v0.10.0",
-    "build_version": "{{ timestamp }}"
+    "version": "0.22.1"
   },
   "builders": [{
     "type": "digitalocean",
-    "snapshot_name": "apollo-mesos-{{user `mesos_version`}}-_amd64_{{user `build_version`}}",
+    "snapshot_name": "apollo-mesos-ubuntu-14.04-amd64",
     "api_token": "{{user `digitalocean_api_token`}}",
     "image": "ubuntu-14-04-x64",
     "region": "{{user `digitalocean_region`}}",
@@ -43,6 +43,28 @@
         "scripts/common/install_weave.sh"
       ],
       "execute_command": "{{ .Vars }} sudo -E -S bash -c '{{ .Path }}'"
+    },
+    {
+      "type": "file",
+      "source": "tests",
+      "destination": "/tmp"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/common/serverspec.sh",
+      "execute_command": "{{ .Vars }} sudo  -E -S bash -c '{{ .Path }}'"
+    }
+  ],
+  "push": {
+    "name": "capgemini/apollo-mesos-ubuntu-14-04-amd64"
+  },
+  "post-processors": [{
+    "type": "atlas",
+    "artifact": "capgemini/apollo-mesos-ubuntu-14.04-amd64",
+    "artifact_type": "digitalocean.image",
+    "metadata": {
+      "created_at": "{{timestamp}}",
+      "version": "{{user `version`}}"
     }
-  ]
+  }]
 }
diff --git a/packer/ubuntu-14.04_amd64-google.json b/packer/ubuntu-14.04_amd64-google.json
@@ -5,12 +5,11 @@
     "mesos_version": "0.22.0-1.0.ubuntu1404",
     "marathon_version": "0.8.1-1.0.171.ubuntu1404",
     "consul_version": "0.5.0",
-    "weave_version": "v0.10.0",
-    "build_version": "{{ timestamp }}"
+    "weave_version": "v0.10.0"
   },
   "builders": [{
     "type": "googlecompute",
-    "image_name": "apollo-ubuntu-mesos-14-04-x64-{{user `build_version`}}",
+    "image_name": "apollo-mesos-ubuntu-14.04-amd64",
     "image_description": "Ubuntu 14.04 LTS, Meosos {{user `mesos_version`}}, Docker, Marathon, Consul {{user `consul_version`}}, Weave {{user `weave_version`}}, and Zookeeper.",
     "ssh_username": "ubuntu",
     "ssh_timeout": "10m",
@@ -55,5 +54,17 @@
       "script": "scripts/common/serverspec.sh",
       "execute_command": "{{ .Vars }} sudo  -E -S bash -c '{{ .Path }}'"
     }
-  ]
+  ],
+  "push": {
+    "name": "capgemini/apollo-mesos-ubuntu-14-04-amd64"
+  },
+  "post-processors": [{
+    "type": "atlas",
+    "artifact": "capgemini/apollo-mesos-ubuntu-14.04-amd64",
+    "artifact_type": "googlecompute.image",
+    "metadata": {
+      "created_at": "{{timestamp}}",
+      "version": "{{user `version`}}"
+    }
+  }]
 }
diff --git a/packer/ubuntu-14.04_amd64.json b/packer/ubuntu-14.04_amd64.json
@@ -8,14 +8,13 @@
     "marathon_version": "0.8.1-1.0.171.ubuntu1404",
     "consul_version": "0.5.0",
     "weave_version": "v0.10.0",
-    "build_version": "{{ timestamp }}",
     "access_token": "{{env `ATLAS_TOKEN`}}"
   },
   "builders": [{
     "type": "virtualbox-iso",
     "guest_os_type": "Ubuntu_64",
     "headless": true,
-    "vm_name": "mesos-{{user `mesos_version`}}_amd64_virtualbox_{{user `build_version`}}",
+    "vm_name": "apollo-mesos-ubuntu-14.04-amd64-virtualbox",
     "disk_size": 40960,
     "vboxmanage": [
       [ "modifyvm", "{{.Name}}", "--memory", "2048" ],
@@ -79,13 +78,13 @@
   "post-processors": [
     [{
       "type": "vagrant",
-      "output": "build/ubuntu-14.04_amd64_{{.Provider}}/mesos-{{user `mesos_version`}}_amd64_{{.Provider}}_{{user `build_version`}}.box",
+      "output": "build/ubuntu-14.04_amd64_{{.Provider}}/apollo-mesos-{{user `mesos_version`}}_amd64_{{.Provider}}.box",
       "keep_input_artifact": true,
       "compression_level": 9
     },
     {
       "type": "vagrant-cloud",
-      "box_tag": "capgemini/mesos",
+      "box_tag": "capgemini/apollo-mesos",
       "access_token": "{{user `access_token`}}",
       "version": "{{user `version`}}"
     }]

diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf
@@ -68,8 +68,8 @@ variable "instance_type" {
 
 variable "atlas_artifact" {
   default = {
-    master = "capgemini/mesos-0.22.1_ubuntu-14.04_amd64"
-    slave  = "capgemini/mesos-0.22.1_ubuntu-14.04_amd64"
+    master = "capgemini/apollo-mesos-ubuntu-14.04-amd64"
+    slave  = "capgemini/apollo-mesos-ubuntu-14.04-amd64"
   }
 }
 

diff --git a/terraform/digitalocean/mesos-masters.tf b/terraform/digitalocean/mesos-masters.tf
@@ -4,9 +4,15 @@ resource "digitalocean_ssh_key" "default" {
     public_key = "${file(var.key_file)}"
 }
 
+/* Base packer build we use for provisioning master instances */
+resource "atlas_artifact" "mesos-master" {
+  name = "${var.atlas_artifact.master}"
+  type = "digitalocean.image"
+}
+
 /* Mesos master instances */
 resource "digitalocean_droplet" "mesos-master" {
-  image              = "${var.image.master}"
+  image              = "${atlas_artifact.mesos-master.id}"
   region             = "${var.region}"
   count              = "${var.masters}"
   name               = "apollo-mesos-master-${count.index}"

diff --git a/terraform/digitalocean/mesos-slaves.tf b/terraform/digitalocean/mesos-slaves.tf
@@ -1,6 +1,12 @@
+/* Base packer build we use for provisioning master instances */
+resource "atlas_artifact" "mesos-slave" {
+  name = "${var.atlas_artifact.slave}"
+  type = "digitalocean.image"
+}
+
 /* Mesos slave instances */
 resource "digitalocean_droplet" "mesos-slave" {
-  image              = "${var.image.slave}"
+  image              = "${atlas_artifact.mesos-slave.id}"
   region             = "${var.region}"
   count              = "${var.slaves}"
   name               = "apollo-mesos-slave-${count.index}"

diff --git a/terraform/digitalocean/variables.tf b/terraform/digitalocean/variables.tf
@@ -23,11 +23,9 @@ variable "instance_size" {
   }
 }
 
-variable "image" {
-  description = "Base Image to launch the droplets with"
+variable "atlas_artifact" {
   default = {
-    master = "11147507"
-    slave  = "11147507"
+    master = "capgemini/apollo-mesos-ubuntu-14.04-amd64"
+    slave  = "capgemini/apollo-mesos-ubuntu-14.04-amd64"
   }
 }
-