Skip to content

Commit

Permalink
Adding better retry logic when checking for existence
Browse files Browse the repository at this point in the history
  • Loading branch information
tyler-ball committed Jul 30, 2015
1 parent 11dd79a commit ab5a8f2
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 15 deletions.
23 changes: 12 additions & 11 deletions lib/kitchen/driver/ec2.rb
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,17 @@ def create(state) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
server = submit_server
end
info("Instance <#{server.id}> requested.")
ec2.client.wait_until(
:instance_exists,
:instance_ids => [server.id]
)
tag_server(server)
# Unfortunately the AWS SDK doesn't actually wait correctly for instance existence, so we
# need to retry if they throw a `server with id doesn't exist` or `id does not exist` error
Retryable.retryable(:tries => 10, :sleep => lambda { |n| [2**n, 30].min }) do |r, _|
debug("Putting existence check in a retryable loop because it can fail, #{r} retries")
server.wait_until_exists do |w|
w.before_attempt do |attempts|
info("Polling AWS for existence, #{attempts} attempt...")
end
end
tag_server(server)
end

state[:server_id] = server.id
info("EC2 instance <#{state[:server_id]}> created.")
Expand Down Expand Up @@ -339,12 +345,7 @@ def tag_server(server)
config[:tags].each do |k, v|
tags << { :key => k, :value => v }
end
# Unfortunately the AWS SDK doesn't actually wait correctly for instance existence, so we
# need to retry if they throw a `server with id doesn't exist` error
Retryable.retryable(:tries => 10, :sleep => lambda { |n| [2**n, 10].min }) do |retries, _|
info "Attempting to tag the server, attempt #{retries}/10"
server.create_tags(:tags => tags)
end
server.create_tags(:tags => tags)
end

def wait_until_ready(server, state)
Expand Down
5 changes: 1 addition & 4 deletions spec/kitchen/driver/ec2_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,7 @@

shared_examples "common create" do
it "successfully creates and tags the instance" do
expect(actual_client).to receive(:wait_until).with(
:instance_exists,
:instance_ids => [server.id]
)
expect(server).to receive(:wait_until_exists)
expect(driver).to receive(:tag_server).with(server)
expect(driver).to receive(:wait_until_ready).with(server, state)
expect(transport).to receive_message_chain("connection.wait_until_ready")
Expand Down

0 comments on commit ab5a8f2

Please sign in to comment.