From ab5a8f2ad8f8a63e2530fb67e3d6e17a4e0d17ea Mon Sep 17 00:00:00 2001 From: tyler-ball Date: Thu, 30 Jul 2015 13:56:49 -0600 Subject: [PATCH] Adding better retry logic when checking for existence --- lib/kitchen/driver/ec2.rb | 23 ++++++++++++----------- spec/kitchen/driver/ec2_spec.rb | 5 +---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/kitchen/driver/ec2.rb b/lib/kitchen/driver/ec2.rb index 1c983a96..10d77455 100644 --- a/lib/kitchen/driver/ec2.rb +++ b/lib/kitchen/driver/ec2.rb @@ -194,11 +194,17 @@ def create(state) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength server = submit_server end info("Instance <#{server.id}> requested.") - ec2.client.wait_until( - :instance_exists, - :instance_ids => [server.id] - ) - tag_server(server) + # Unfortunately the AWS SDK doesn't actually wait correctly for instance existence, so we + # need to retry if they throw a `server with id doesn't exist` or `id does not exist` error + Retryable.retryable(:tries => 10, :sleep => lambda { |n| [2**n, 30].min }) do |r, _| + debug("Putting existence check in a retryable loop because it can fail, #{r} retries") + server.wait_until_exists do |w| + w.before_attempt do |attempts| + info("Polling AWS for existence, #{attempts} attempt...") + end + end + tag_server(server) + end state[:server_id] = server.id info("EC2 instance <#{state[:server_id]}> created.") @@ -339,12 +345,7 @@ def tag_server(server) config[:tags].each do |k, v| tags << { :key => k, :value => v } end - # Unfortunately the AWS SDK doesn't actually wait correctly for instance existence, so we - # need to retry if they throw a `server with id doesn't exist` error - Retryable.retryable(:tries => 10, :sleep => lambda { |n| [2**n, 10].min }) do |retries, _| - info "Attempting to tag the server, attempt #{retries}/10" - server.create_tags(:tags => tags) - end + server.create_tags(:tags => tags) end def wait_until_ready(server, state) diff --git a/spec/kitchen/driver/ec2_spec.rb b/spec/kitchen/driver/ec2_spec.rb index aa106cde..a7db9140 100644 --- a/spec/kitchen/driver/ec2_spec.rb +++ b/spec/kitchen/driver/ec2_spec.rb @@ -337,10 +337,7 @@ shared_examples "common create" do it "successfully creates and tags the instance" do - expect(actual_client).to receive(:wait_until).with( - :instance_exists, - :instance_ids => [server.id] - ) + expect(server).to receive(:wait_until_exists) expect(driver).to receive(:tag_server).with(server) expect(driver).to receive(:wait_until_ready).with(server, state) expect(transport).to receive_message_chain("connection.wait_until_ready")