From 1a37a3b57ff66e7746419c12871a0d83fe6ff123 Mon Sep 17 00:00:00 2001 From: Wayne Robinson Date: Thu, 22 Feb 2018 14:49:28 +1100 Subject: [PATCH 1/2] Allow Batch Size to be Specified for Requeue Sometimes batches of 10 messages will be larger than the 256KB per-request limit of batch send so allow this to be specified on the command-line during re-queue. A better (future) option would be to automatically detect the message size and only send as many messages as will fit in a batch send request. --- bin/cli/sqs.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/cli/sqs.rb b/bin/cli/sqs.rb index 6f0a9071..34c71772 100644 --- a/bin/cli/sqs.rb +++ b/bin/cli/sqs.rb @@ -43,8 +43,8 @@ def batch_delete(url, messages) end end - def batch_send(url, messages) - messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(10) do |batch| + def batch_send(url, messages, messages_per_batch = 10) + messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch| sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure| say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow end @@ -152,12 +152,13 @@ def dump(queue_name) end desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file' + method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number messages per batch to send' def requeue(queue_name, path) fail_task "Path #{path} not found" unless File.exist?(path) messages = File.readlines(path).map { |line| JSON.parse(line, symbolize_names: true) } - batch_send(find_queue_url(queue_name), messages) + batch_send(find_queue_url(queue_name), messages, options[:batch_size]) say "Requeued #{messages.size} messages from #{path} to #{queue_name}", :green end From d057681a05eb5f27d5908767593d48d68d2547d5 Mon Sep 17 00:00:00 2001 From: Pablo Cantero Date: Mon, 26 Feb 2018 11:26:24 -0500 Subject: [PATCH 2/2] Update sqs.rb --- bin/cli/sqs.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cli/sqs.rb b/bin/cli/sqs.rb index 34c71772..0547b526 100644 --- a/bin/cli/sqs.rb +++ b/bin/cli/sqs.rb @@ -152,7 +152,7 @@ def dump(queue_name) end desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file' - method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number messages per batch to send' + method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send' def requeue(queue_name, path) fail_task "Path #{path} not found" unless File.exist?(path)