Use aws-sdk v3

* introduce individual aws gems as dependencies * no more eager_autoload
logstash-plugins · Jun 23, 2022 · 80cdd69 · 80cdd69
1 parent 8ce489a
commit 80cdd69
Show file tree

Hide file tree

Showing 24 changed files with 18 additions and 100 deletions.
diff --git a/lib/logstash/inputs/cloudwatch.rb b/lib/logstash/inputs/cloudwatch.rb
@@ -4,7 +4,7 @@
 require "logstash/plugin_mixins/aws_config"
 require "logstash/util"
 require "stud/interval"
-require "aws-sdk"
+require "aws-sdk-cloudwatch"
 
 # Pull events from the Amazon Web Services CloudWatch API.
 #

diff --git a/lib/logstash/inputs/s3.rb b/lib/logstash/inputs/s3.rb
@@ -7,13 +7,11 @@
 require "tmpdir"
 require "stud/interval"
 require "stud/temporary"
-require "aws-sdk"
-require "logstash/inputs/s3/patch"
+require "aws-sdk-s3"
 require "logstash/plugin_mixins/ecs_compatibility_support"
 
 require 'java'
 
-Aws.eager_autoload!
 # Stream events from files from a S3 bucket.
 #
 # Each line from each file generates an event.
@@ -99,7 +97,6 @@ def initialize(*params)
   def register
     require "fileutils"
     require "digest/md5"
-    require "aws-sdk-resources"
 
     @logger.info("Registering", :bucket => @bucket, :region => @region)
 

diff --git a/lib/logstash/inputs/s3/patch.rb b/lib/logstash/inputs/s3/patch.rb
diff --git a/lib/logstash/inputs/sqs.rb b/lib/logstash/inputs/sqs.rb
@@ -5,13 +5,6 @@
 require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
 require "logstash/errors"
-require 'logstash/inputs/sqs/patch'
-
-# Forcibly load all modules marked to be lazily loaded.
-#
-# It is recommended that this is called prior to launching threads. See
-# https://aws.amazon.com/blogs/developer/threading-with-the-aws-sdk-for-ruby/.
-Aws.eager_autoload!
 
 # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
 #
@@ -103,7 +96,7 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
   attr_reader :poller
 
   def register
-    require "aws-sdk"
+    require "aws-sdk-sqs"
     @logger.info("Registering SQS input", :queue => @queue, :queue_owner_aws_account_id => @queue_owner_aws_account_id)
 
     setup_queue

diff --git a/lib/logstash/inputs/sqs/patch.rb b/lib/logstash/inputs/sqs/patch.rb
diff --git a/lib/logstash/outputs/cloudwatch.rb b/lib/logstash/outputs/cloudwatch.rb
@@ -161,7 +161,7 @@ class LogStash::Outputs::CloudWatch < LogStash::Outputs::Base
   public
   def register
     require "thread"
-    require "aws-sdk"
+    require "aws-sdk-cloudwatch"
 
     @cw = Aws::CloudWatch::Client.new(aws_options_hash)
 

diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb
@@ -11,10 +11,7 @@
 require "fileutils"
 require "set"
 require "pathname"
-require "aws-sdk"
-require "logstash/outputs/s3/patch"
-
-Aws.eager_autoload!
+require "aws-sdk-s3"
 
 # INFORMATION:
 #

diff --git a/lib/logstash/outputs/s3/patch.rb b/lib/logstash/outputs/s3/patch.rb
diff --git a/lib/logstash/outputs/s3/uploader.rb b/lib/logstash/outputs/s3/uploader.rb
@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "logstash/util"
-require "aws-sdk"
+require "aws-sdk-core"
 
 module LogStash
   module Outputs

diff --git a/lib/logstash/outputs/sns.rb b/lib/logstash/outputs/sns.rb
@@ -56,7 +56,7 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base
 
   public
   def register
-    require "aws-sdk-resources"
+    require "aws-sdk-sns"
 
     @sns = Aws::SNS::Client.new(aws_options_hash)
 

diff --git a/lib/logstash/outputs/sqs.rb b/lib/logstash/outputs/sqs.rb
@@ -1,13 +1,11 @@
 # encoding: utf-8
 
-require 'aws-sdk'
+require 'aws-sdk-sqs'
 require 'logstash/errors'
 require 'logstash/namespace'
 require 'logstash/outputs/base'
 require 'logstash/plugin_mixins/aws_config'
 
-Aws.eager_autoload!
-
 # Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue.
 #
 # SQS is a simple, scalable queue system that is part of the Amazon Web

diff --git a/logstash-integration-aws.gemspec b/logstash-integration-aws.gemspec
@@ -34,7 +34,13 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "rufus-scheduler", ">= 3.0.9"
   s.add_runtime_dependency "stud", "~> 0.0.22"
-  s.add_runtime_dependency "aws-sdk", "~> 2"
+  s.add_runtime_dependency "aws-sdk-core", "~> 3"
+  s.add_runtime_dependency "aws-sdk-s3"
+  s.add_runtime_dependency "aws-sdk-sqs"
+  s.add_runtime_dependency "aws-sdk-sns"
+  s.add_runtime_dependency "aws-sdk-cloudwatch"
+  s.add_runtime_dependency "aws-sdk-cloudfront"
+  s.add_runtime_dependency "aws-sdk-resourcegroups"
 
   s.add_development_dependency "logstash-codec-json_lines"
   s.add_development_dependency "logstash-codec-multiline"

diff --git a/spec/inputs/cloudwatch_spec.rb b/spec/inputs/cloudwatch_spec.rb
@@ -1,7 +1,6 @@
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/devutils/rspec/shared_examples'
 require 'logstash/inputs/cloudwatch'
-require 'aws-sdk'
 
 describe LogStash::Inputs::CloudWatch do
   subject { LogStash::Inputs::CloudWatch.new(config) }

diff --git a/spec/inputs/s3_spec.rb b/spec/inputs/s3_spec.rb
@@ -4,10 +4,8 @@
 require "logstash/inputs/s3"
 require "logstash/codecs/multiline"
 require "logstash/errors"
-require "aws-sdk-resources"
 require "stud/temporary"
 require_relative "../support/helpers"
-require "aws-sdk"
 require "fileutils"
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 

diff --git a/spec/inputs/sqs_spec.rb b/spec/inputs/sqs_spec.rb
@@ -5,7 +5,7 @@
 require "logstash/errors"
 require "logstash/event"
 require "logstash/json"
-require "aws-sdk"
+require "aws-sdk-sqs"
 require "ostruct"
 
 describe LogStash::Inputs::SQS do

diff --git a/spec/integration/cloudwatch_spec.rb b/spec/integration/cloudwatch_spec.rb
@@ -1,6 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/cloudwatch"
-require "aws-sdk"
 
 describe LogStash::Inputs::CloudWatch, :integration => true do
 

diff --git a/spec/integration/s3_spec.rb b/spec/integration/s3_spec.rb
@@ -1,6 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/s3"
-require "aws-sdk"
 require "fileutils"
 require_relative "../support/helpers"
 

diff --git a/spec/integration/sqs_spec.rb b/spec/integration/sqs_spec.rb
@@ -3,7 +3,6 @@
 require "logstash/inputs/sqs"
 require "logstash/event"
 require "logstash/json"
-require "aws-sdk"
 require_relative "../support/helpers"
 require "thread"
 

diff --git a/spec/outputs/s3/uploader_spec.rb b/spec/outputs/s3/uploader_spec.rb
@@ -2,7 +2,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/s3/uploader"
 require "logstash/outputs/s3/temporary_file"
-require "aws-sdk"
 require "stud/temporary"
 
 describe LogStash::Outputs::S3::Uploader do

diff --git a/spec/outputs/s3/write_bucket_permission_validator_spec.rb b/spec/outputs/s3/write_bucket_permission_validator_spec.rb
@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/s3/write_bucket_permission_validator"
-require "aws-sdk"
 
 describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
   let(:logger) { spy(:logger ) }

diff --git a/spec/outputs/sns_spec.rb b/spec/outputs/sns_spec.rb
@@ -4,7 +4,7 @@
 require 'logstash/event'
 require "logstash/plugin_mixins/aws_config"
 
-require "aws-sdk" # TODO: Why is this not automatically brought in by the aws_config plugin?
+require "aws-sdk-sns"
 
 describe LogStash::Outputs::Sns do
   let(:arn) { "arn:aws:sns:us-east-1:999999999:logstash-test-sns-topic" }

diff --git a/spec/plugin_mixin/aws_config_spec.rb b/spec/plugin_mixin/aws_config_spec.rb
@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/plugin_mixins/aws_config"
-require 'aws-sdk'
 require 'timecop'
 
 class DummyInputAwsConfigV2 < LogStash::Inputs::Base

diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
@@ -1,6 +1,5 @@
 # encoding: utf-8
 
-require 'aws-sdk'
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/outputs/sqs'
 require 'logstash/logging/logger'

diff --git a/spec/unit/outputs/sqs_spec.rb b/spec/unit/outputs/sqs_spec.rb
@@ -194,7 +194,7 @@
         # overhead is caused by metadata (such as the `message` field name and
         # the `@timestamp` field) as well as additional characters as a result
         # of encoding the event.
-        overhead = 69
+        overhead = 90
 
         events = [
           LogStash::Event.new('message' => 'a' * (0.6 * message_max_size - overhead)),