rescue all exceptions for metrics

lib/sidekiq/instrument/middleware/server.rb

@@ -17,15 +17,21 @@ def call(worker, job, _queue, &block)
       execution_time_ms = (Time.now - start_time) * 1000
       Statter.statsd.measure(metric_name(worker, 'runtime'), execution_time_ms)
       Statter.dogstatsd&.timing('sidekiq.runtime', execution_time_ms, worker_dog_options(worker))
-    rescue StandardError => e
+    rescue Exception => e
+      dd_options = worker_dog_options(worker)
+      dd_options[:tags] << "error:#{e.class.name}"
+
       # if we have retries left, increment the enqueue.retry counter to indicate the job is going back on the queue
       if max_retries(worker) > current_retries(job) + 1
         WorkerMetrics.trace_workers_increment_counter(worker.class.to_s.underscore)
-        Statter.dogstatsd&.increment('sidekiq.enqueue.retry', worker_dog_options(worker))
+        Statter.dogstatsd&.increment('sidekiq.enqueue.retry', dd_options)
       end
 
+      # categorize rate limit lock errors differently from actual errors
+      error_string = e.class.name.eql?("RedisRateLimit::Throttle::LockFailedError") ? 'sidekiq.error.redis_rate_lock' : 'sidekiq.error'
+      Statter.dogstatsd&.increment(error_string, dd_options)
       Statter.statsd.increment(metric_name(worker, 'error'))
-      Statter.dogstatsd&.increment('sidekiq.error', worker_dog_options(worker))
+
       raise e
     ensure
       WorkerMetrics.trace_workers_decrement_counter(worker.class.to_s.underscore)

lib/sidekiq/instrument/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Instrument
-    VERSION = '0.7.1'
+    VERSION = '0.7.2'
   end
 end

spec/sidekiq-instrument/server_middleware_spec.rb

@@ -5,6 +5,8 @@
 RSpec.describe Sidekiq::Instrument::ServerMiddleware do
   describe '#call' do
     let(:expected_dog_options) { { tags: ['queue:default', 'worker:my_worker'] } }
+    let(:expected_error_dog_options) { { tags: ['queue:default', 'worker:my_worker', 'error:RuntimeError'] } }
+
     let(:worker_metric_name) do
       'sidekiq_instrument_trace_workers::in_queue'
     end
@@ -83,7 +85,7 @@
     context 'when a retried job succeeds' do
       before do
         Sidekiq[:max_retries] = 1
-        allow_any_instance_of(MyWorker).to receive(:perform).and_raise('foo')
+        allow_any_instance_of(MyWorker).to receive(:perform).and_raise(RuntimeError.new('foo'))
 
         # This makes the job look like a retry since we can't access the job argument
         allow_any_instance_of(Sidekiq::Instrument::ServerMiddleware).to receive(:current_retries).and_return(0)
@@ -111,7 +113,7 @@
     context 'when a job fails' do
       before do
         Sidekiq[:max_retries] = 0
-        allow_any_instance_of(MyWorker).to receive(:perform).and_raise('foo')
+        allow_any_instance_of(MyWorker).to receive(:perform).and_raise(RuntimeError.new('foo'))
       end
 
       it 'increments the StatsD error counter' do
@@ -129,7 +131,7 @@
         expect(Sidekiq::Instrument::Statter.dogstatsd).not_to receive(:time)
         expect(
           Sidekiq::Instrument::Statter.dogstatsd
-        ).to receive(:increment).with('sidekiq.error', expected_dog_options).once
+        ).to receive(:increment).with('sidekiq.error', expected_error_dog_options).once
 
         begin
           MyWorker.perform_async
@@ -151,11 +153,11 @@
             ).to receive(:increment).with('sidekiq.dequeue', expected_dog_options).once
             expect(
               Sidekiq::Instrument::Statter.dogstatsd
-            ).not_to receive(:increment).with('sidekiq.enqueue.retry', expected_dog_options)
+            ).not_to receive(:increment).with('sidekiq.enqueue.retry', expected_error_dog_options)
             expect(Sidekiq::Instrument::Statter.dogstatsd).not_to receive(:time)
             expect(
               Sidekiq::Instrument::Statter.dogstatsd
-            ).to receive(:increment).with('sidekiq.error', expected_dog_options).once
+            ).to receive(:increment).with('sidekiq.error', expected_error_dog_options).once
 
             begin
               MyWorker.perform_async
@@ -185,11 +187,11 @@
             ).to receive(:increment).with('sidekiq.dequeue', expected_dog_options).once
             expect(
               Sidekiq::Instrument::Statter.dogstatsd
-            ).to receive(:increment).with('sidekiq.enqueue.retry', expected_dog_options).once
+            ).to receive(:increment).with('sidekiq.enqueue.retry', expected_error_dog_options).once
             expect(Sidekiq::Instrument::Statter.dogstatsd).not_to receive(:time)
             expect(
               Sidekiq::Instrument::Statter.dogstatsd
-            ).to receive(:increment).with('sidekiq.error', expected_dog_options).once
+            ).to receive(:increment).with('sidekiq.error', expected_error_dog_options).once
 
             begin
               MyWorker.perform_async
@@ -223,7 +225,38 @@
           expect(Sidekiq::Instrument::Statter.dogstatsd).not_to receive(:time)
           expect(
             Sidekiq::Instrument::Statter.dogstatsd
-          ).to receive(:increment).with('sidekiq.error', expected_dog_options).once
+          ).to receive(:increment).with('sidekiq.error', expected_error_dog_options).once
+
+          begin
+            MyWorker.perform_async
+          rescue StandardError
+            nil
+          end
+        end
+      end
+
+      context 'when the error is RedisRateLimit::Throttle::LockFailedError' do
+        let(:expected_lock_error_dog_options) { { tags: ['queue:default', 'worker:my_worker', 'error:RedisRateLimit::Throttle::LockFailedError'] } }
+        let(:lock_error) { RuntimeError.new('foo') }
+
+        before do
+          # force the error's class name to be the RedisRateLimit error type string
+          # it's an external error class and we can't actually create an instance of the error
+          allow(lock_error).to receive_message_chain(:class, :name).and_return("RedisRateLimit::Throttle::LockFailedError")
+          allow_any_instance_of(MyWorker).to receive(:perform).and_raise(lock_error)
+        end
+
+        it 'increments the DogStatsD sidekiq.error.redis_rate_lock counter' do
+          expect(
+            Sidekiq::Instrument::Statter.dogstatsd
+          ).to receive(:increment).with('sidekiq.dequeue', expected_dog_options).once
+          expect(
+            Sidekiq::Instrument::Statter.dogstatsd
+          ).not_to receive(:increment).with('sidekiq.enqueue.retry', expected_lock_error_dog_options)
+          expect(Sidekiq::Instrument::Statter.dogstatsd).not_to receive(:time)
+          expect(
+            Sidekiq::Instrument::Statter.dogstatsd
+          ).to receive(:increment).with('sidekiq.error.redis_rate_lock', expected_lock_error_dog_options).once
 
           begin
             MyWorker.perform_async