White 13 05 acu114901 implement ephemeral library cookbook

.kitchen.yml

@@ -1,19 +1,68 @@
 ---
 driver_plugin: vagrant
 driver_config:
-  require_chef_omnibus: false
+  require_chef_omnibus: true
 
 platforms:
-- name: rightimage-ubuntu-12.04
+- name: ubuntu-12.04
   driver_config:
-    box: RightImage_Ubuntu_12.04_x64_v13.5.0.1
-    box_url: https://rightscale-vagrant.s3.amazonaws.com/virtualbox/ubuntu/12.04/RightImage_Ubuntu_12.04_x64_v13.5.0.1.box
-- name: rightimage-centos-6.4
+    box: opscode-ubuntu-12.04
+    box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_ubuntu-12.04_provisionerless.box
+- name: ubuntu-10.04
   driver_config:
-    box: RightImage_CentOS_6.4_x64_v13.5.0.1
-    box_url: https://rightscale-vagrant.s3.amazonaws.com/virtualbox/centos/6.4/RightImage_CentOS_6.4_x64_v13.5.0.1.box
+    box: opscode-ubuntu-10.04
+    box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_ubuntu-10.04_provisionerless.box
+- name: centos-6.4
+  driver_config:
+    box: opscode-centos-6.4
+    box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_centos-6.4_provisionerless.box
+- name: centos-5.9
+  driver_config:
+    box: opscode-centos-5.9
+    box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_centos-5.9_provisionerless.box
 
 suites:
+# This test suite mimics the behavior of being in a cloud which supports block device mapping
+# The loopback devices given in block_device_mapping_ephemeral<N> should be included in
+# the test/cookbooks/fake cookbook's attributes so they get created while running
+# the test. EC2 and openstack clouds follow this format.
+#
 - name: default
-  run_list: ["recipe[ephemeral]"]
-  attributes: {}
+  run_list: ["recipe[fake]", "recipe[ephemeral_lvm]"]
+  attributes:
+    cloud:
+      provider: vagrant
+    ephemeral_lvm:
+      filesystem: ext3
+    vagrant:
+      block_device_mapping_ephemeral0: loop0
+      block_device_mapping_ephemeral1: loop1
+
+# This test mimics the behavior of being in a google compute engine. The ephemeral disks
+# are mapped to /dev/disk/by-id/google-ephemeral-disk-<N>. Loopback device matching the
+# device index should be set in the test/cookbooks/fake cookbook's attributes
+# so they get created. For example, the following attributes will require the creation of
+# loopback devices "/dev/loop0" and "/dev/loop1"
+#
+- name: gce
+  run_list: ["recipe[fake::gce]", "recipe[ephemeral_lvm]"]
+  attributes:
+    cloud:
+      provider: gce
+    ephemeral_lvm:
+      filesystem: ext3
+    gce:
+      attached_disks:
+        disks:
+        - deviceName: boot
+          index: 0
+          mode: READ_WRITE
+          type: EPHEMERAL
+        - deviceName: ephemeral-disk-0
+          index: 1
+          mode: READ_WRITE
+          type: EPHEMERAL
+        - deviceName: ephemeral-disk-1
+          index: 2
+          mode: READ_WRITE
+          type: EPHEMERAL

Berksfile

@@ -1,3 +1,5 @@
 site :opscode
 
 metadata
+
+cookbook "fake", path: "./test/cookbooks/fake", group: :integration

Gemfile

@@ -1,7 +1,20 @@
 source 'https://rubygems.org'
 
+# Berkshelf had some issues with latest version of Celluloid
+# which required latest version of ridley which was causing problems.
+# So locking ridley to '~> 1.5.0' here makes things work
+#
+gem 'ridley', '~> 1.5.0'
 gem 'berkshelf'
 gem 'thor-foodcritic'
 gem 'thor-scmversion'
-gem 'test-kitchen', '~> 1.0.0.beta.2', :group => :integration
-gem 'kitchen-vagrant', :group => :integration
+
+group :integration do
+  gem 'test-kitchen', '~> 1.0.0.beta.3'
+  gem 'kitchen-vagrant'
+end
+
+group :test do
+  gem 'chefspec', '~> 1.3'
+  gem 'strainer', '~> 3.0'
+end

README.md

@@ -1,15 +1,67 @@
-# ephemeral cookbook
+# ephemeral_lvm cookbook
+
+This cookbook will identify the ephemeral devices available on the instance based on Ohai data. If no ephemeral devices
+are found, it will gracefully exit with a log message. If ephemeral devices are found, they will be setup to
+use LVM and a logical volume will be created, formatted, and mounted. If multiple ephemeral devices are found
+(e.g. m1.large on EC2 has 2 ephemeral devices with 420 GB each), they will be striped to create the LVM.
 
 # Requirements
 
-# Usage
+* Chef 10 or higher
+* A cloud that supports ephemeral devices. Currently supported clouds: EC2, Openstack, and Google.
+* The [lvm](http://community.opscode.com/cookbooks/lvm) cookbook
 
 # Attributes
 
+The following are the attributes used by the this cookbook.
+<table>
+  <tr>
+    <th>Name</th>
+    <th>Description</th>
+    <th>Default</th>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['filesystem']</tt></td>
+    <td>The filesystem to be used on the ephemeral volume</td>
+    <td><tt>'ext4'</tt></td>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['mount_point']</tt></td>
+    <td>The mount point for the ephemeral volume</td>
+    <td><tt>'/mnt/ephemeral'</tt></td>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['volume_group_name']</tt></td>
+    <td>The volume group name for the ephemeral LVM</td>
+    <td><tt>'vg-data'</tt></td>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['logical_volume_size']</tt></td>
+    <td>The size to be used for the ephemeral LVM</td>
+    <td><tt>'100%VG'</tt> - This will use all available space in the volume group</td>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['logical_volume_name']</tt></td>
+    <td>The name of the logical volume for ephemeral LVM</td>
+    <td><tt>'ephemeral0'</tt></td>
+  </tr>
+  <tr>
+    <td><tt>node['ephemeral_lvm']['stripe_size']</tt></td>
+    <td>The stripe size to be used for the ephemeral logical volume</td>
+    <td><tt>512</tt></td>
+  </tr>
+</table>
+
+# Usage
+
+Place the `ephemeral_lvm::default` in the runlist and the ephemeral devices will be setup.
+
 # Recipes
 
+## default
+
+This recipe sets up available ephemeral devices to be an LVM device, formats it, and mounts it.
+
 # Author
 
 Author:: RightScale, Inc. (<cookbooks@rightscale.com>)
-
-Maintained by the RightScale White Team

Strainerfile

@@ -0,0 +1,4 @@
+knife: bundle exec knife cookbook test $COOKBOOK
+foodcritic: bundle exec foodcritic --epic-fail any $SANDBOX/$COOKBOOK
+rspec: bundle exec rspec --color --format documentation
+kitchen: bundle exec kitchen test

Vagrantfile

@@ -6,14 +6,14 @@ Vagrant.configure("2") do |config|
   # options are documented and commented below. For a complete reference,
   # please see the online documentation at vagrantup.com.
 
-  config.vm.hostname = "ephemeral-berkshelf"
+  config.vm.hostname = "ephemeral-lvm-berkshelf"
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "RightImage_Ubuntu_12.04_x64_v13.5.0.1"
+  config.vm.box = "opscode-ubuntu-12.04"
 
   # The url from where the 'config.vm.box' box will be fetched if it
   # doesn't already exist on the user's system.
-  config.vm.box_url = "https://rightscale-vagrant.s3.amazonaws.com/virtualbox/ubuntu/12.04/RightImage_Ubuntu_12.04_x64_v13.5.0.1.box"
+  config.vm.box_url = "https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_ubuntu-12.04_provisionerless.box"
 
   # Assign this VM to a host-only network IP, allowing you to access it
   # via the IP. Host-only networks can talk to the host machine as well as
@@ -70,6 +70,8 @@ Vagrant.configure("2") do |config|
   # to skip installing and copying to Vagrant's shelf.
   # config.berkshelf.except = []
 
+  config.omnibus.chef_version = :latest
+
   config.vm.provision :chef_solo do |chef|
     chef.json = {
       :mysql => {
@@ -80,7 +82,7 @@ Vagrant.configure("2") do |config|
     }
 
     chef.run_list = [
-        "recipe[ephemeral::default]"
+        "recipe[ephemeral_lvm::default]"
     ]
   end
 end

attributes/default.rb

@@ -0,0 +1,36 @@
+#
+# Cookbook Name:: ephemeral_lvm
+# Attributes:: default
+#
+# Copyright (C) 2013 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The ephemeral file system
+default['ephemeral_lvm']['filesystem'] = "ext4"
+
+# The ephemeral mount point
+default['ephemeral_lvm']['mount_point'] = "/mnt/ephemeral"
+
+# The ephemeral volume group name
+default['ephemeral_lvm']['volume_group_name'] = "vg-data"
+
+# The logical volume size of the ephemeral disk
+default['ephemeral_lvm']['logical_volume_size'] = "100%VG"
+
+# The ephemeral logical volume name
+default['ephemeral_lvm']['logical_volume_name'] = "ephemeral0"
+
+# The stripe size in kilobytes to be used if more than one ephemeral disk is found
+default['ephemeral_lvm']['stripe_size'] = 512

libraries/helper.rb

@@ -0,0 +1,104 @@
+#
+# Cookbook Name:: ephemeral_lvm
+# Library:: helper
+#
+# Copyright (C) 2013 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module EphemeralLvm
+  module Helper
+    # Identifies the ephemeral devices available on a cloud server based on cloud-specific Ohai data and returns
+    # them as an array. This method also does the mapping required for Xen hypervisors (/dev/sdX -> /dev/xvdX).
+    #
+    # @param cloud [String] the name of cloud
+    # @param node [Chef::Node] the Chef node
+    #
+    # @return [Array<String>] list of ephemeral available ephemeral devices.
+    #
+    def self.get_ephemeral_devices(cloud, node)
+      ephemeral_devices = []
+      # Detects the ephemeral disks available on the instance.
+      #
+      # If the cloud plugin supports block device mapping on the node, obtain the
+      # information from the node for setting up block device
+      #
+      if node[cloud].keys.any? { |key| key.match(/^block_device_mapping_ephemeral\d+$/) }
+        ephemeral_devices = node[cloud].map do |key, device|
+          if key.match(/^block_device_mapping_ephemeral\d+$/)
+            device.match(/\/dev\//) ? device : "/dev/#{device}"
+          end
+        end
+
+        # Removes nil elements from the ephemeral_devices array if any.
+        ephemeral_devices.compact!
+
+        # Servers running on Xen hypervisor require the block device to be in /dev/xvdX instead of /dev/sdX
+        if node.attribute?('virtualization') && node['virtualization']['system'] == "xen"
+          log "Mapping for devices: #{ephemeral_devices.inspect}"
+          ephemeral_devices = EphemeralLvm::Helper.fix_device_mapping(
+            ephemeral_devices,
+            node['block_device'].keys
+          )
+          Chef::Log.info "Ephemeral disks found for cloud '#{cloud}': #{ephemeral_devices.inspect}"
+        end
+      else
+        # Cloud specific ephemeral detection logic if the cloud doesn't support block_device_mapping
+        #
+        case cloud
+        when 'gce'
+          # According to the GCE documentation, the instances have links for ephemeral disks as
+          # /dev/disk/by-id/google-ephemeral-disk-*. Refer to
+          # https://developers.google.com/compute/docs/disks#scratchdisks for more information.
+          #
+          ephemeral_devices = node[cloud]['attached_disks']['disks'].map do |disk|
+            if disk['type'] == "EPHEMERAL" && disk['deviceName'].match(/^ephemeral-disk-\d+$/)
+              "/dev/disk/by-id/google-#{disk["deviceName"]}"
+            end
+          end
+          # Removes nil elements from the ephemeral_devices array if any.
+          ephemeral_devices.compact!
+        else
+          Chef::Log.info "Cloud '#{cloud}' is not supported by 'ephemeral_lvm' cookbook."
+        end
+      end
+      ephemeral_devices
+    end
+
+    # Fixes the device mapping on Xen hypervisors. When using Xen hypervisors, the devices are mapped from /dev/sdX to
+    # /dev/xvdX. This method will identify if mapping is required (by checking the existence of unmapped device) and
+    # map the devices accordingly.
+    #
+    # @param devices [Array<String>] list of devices to fix the mapping
+    # @param node_block_devices [Array<String>] list of block devices currently attached to the server
+    #
+    # @return [Array<String>] list of devices with fixed mapping
+    #
+    def self.fix_device_mapping(devices, node_block_devices)
+      devices.map! do |device|
+        if node_block_devices.include?(device.match(/\/dev\/([a-z]+)$/)[1])
+          device
+        else
+          fixed_device = device.sub("/sd", "/xvd")
+          if node_block_devices.include?(fixed_device.match(/\/dev\/([a-z]+)$/)[1])
+            fixed_device
+          else
+            Chef::Log.warn "could not find ephemeral device: #{device}"
+          end
+        end
+      end
+      devices.compact
+    end
+  end
+end