From d39dceb7479c3351fd90577ce94239d1166a87b1 Mon Sep 17 00:00:00 2001 From: Max Prokopiev Date: Tue, 7 Jun 2022 16:20:57 +0200 Subject: [PATCH 1/2] Add support for the RateLimit-Reset header --- .rubocop_todo.yml | 9 +++++-- lib/faraday/retry/middleware.rb | 36 +++++++++++++++++---------- spec/faraday/retry/middleware_spec.rb | 18 ++++++++++++-- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 932c4be..3cda7d4 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2022-01-04 18:53:25 UTC using RuboCop version 1.23.0. +# on 2022-06-07 14:20:20 UTC using RuboCop version 1.21.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -17,6 +17,11 @@ Lint/AmbiguousBlockAssociation: Metrics/AbcSize: Max: 26 +# Offense count: 1 +# Configuration parameters: CountComments, CountAsOne. +Metrics/ClassLength: + Max: 105 + # Offense count: 2 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods. Metrics/MethodLength: @@ -49,7 +54,7 @@ RSpec/InstanceVariable: RSpec/MultipleExpectations: Max: 3 -# Offense count: 9 +# Offense count: 11 # Configuration parameters: AllowSubject. RSpec/MultipleMemoizedHelpers: Max: 9 diff --git a/lib/faraday/retry/middleware.rb b/lib/faraday/retry/middleware.rb index 34ea311..cfad003 100644 --- a/lib/faraday/retry/middleware.rb +++ b/lib/faraday/retry/middleware.rb @@ -126,7 +126,7 @@ def initialize(app, options = nil) end def calculate_sleep_amount(retries, env) - retry_after = calculate_retry_after(env) + retry_after = [calculate_retry_after(env), calculate_rate_limit_reset(env)].compact.max retry_interval = calculate_retry_interval(retries) return if retry_after && retry_after > @options.max_interval @@ -212,21 +212,16 @@ def rewind_files(body) end end + # RFC for RateLimit Header Fields for HTTP: + # https://tools.ietf.org/id/draft-polli-ratelimit-headers-00.html#rfc.section.3.3 + def calculate_rate_limit_reset(env) + parse_retry_header(env, 'RateLimit-Reset') + end + # MDN spec for Retry-After header: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After def calculate_retry_after(env) - response_headers = env[:response_headers] - return unless response_headers - - retry_after_value = env[:response_headers]['Retry-After'] - - # Try to parse date from the header value - begin - datetime = DateTime.rfc2822(retry_after_value) - datetime.to_time - Time.now.utc - rescue ArgumentError - retry_after_value.to_f - end + parse_retry_header(env, 'Retry-After') end def calculate_retry_interval(retries) @@ -239,6 +234,21 @@ def calculate_retry_interval(retries) current_interval + random_interval end + + def parse_retry_header(env, header) + response_headers = env[:response_headers] + return unless response_headers + + retry_after_value = env[:response_headers][header] + + # Try to parse date from the header value + begin + datetime = DateTime.rfc2822(retry_after_value) + datetime.to_time - Time.now.utc + rescue ArgumentError + retry_after_value.to_f + end + end end end end diff --git a/spec/faraday/retry/middleware_spec.rb b/spec/faraday/retry/middleware_spec.rb index 6539077..f6f4863 100644 --- a/spec/faraday/retry/middleware_spec.rb +++ b/spec/faraday/retry/middleware_spec.rb @@ -245,8 +245,15 @@ conn.get('/unstable') end - context 'when retry_after bigger than interval' do - let(:headers) { { 'Retry-After' => '0.5' } } + context 'when Retry-After bigger than RateLimit-Reset' do + let(:headers) { { 'Retry-After' => '0.5', 'RateLimit-Reset' => '0.1' } } + let(:options) { [{ max: 1, interval: 0.1, retry_statuses: 504 }] } + + it { expect(elapsed).to be > 0.5 } + end + + context 'when RateLimit-Reset bigger than Retry-After' do + let(:headers) { { 'Retry-After' => '0.1', 'RateLimit-Reset' => '0.5' } } let(:options) { [{ max: 1, interval: 0.1, retry_statuses: 504 }] } it { expect(elapsed).to be > 0.5 } @@ -259,6 +266,13 @@ it { expect(elapsed).to be > 0.2 } end + context 'when RateLimit-Reset is a timestamp' do + let(:headers) { { 'Retry-After' => '0.1', 'RateLimit-Reset' => (Time.now.utc + 2).strftime('%a, %d %b %Y %H:%M:%S GMT') } } + let(:options) { [{ max: 1, interval: 0.1, retry_statuses: 504 }] } + + it { expect(elapsed).to be > 1 } + end + context 'when retry_after is a timestamp' do let(:headers) { { 'Retry-After' => (Time.now.utc + 2).strftime('%a, %d %b %Y %H:%M:%S GMT') } } let(:options) { [{ max: 1, interval: 0.1, retry_statuses: 504 }] } From df7f0f39182f612ca892e8573499937445314295 Mon Sep 17 00:00:00 2001 From: Max Prokopiev Date: Tue, 7 Jun 2022 17:19:35 +0200 Subject: [PATCH 2/2] Update README with info about RateLimit-xxx headers --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c8ed0ee..570460f 100644 --- a/README.md +++ b/README.md @@ -96,11 +96,14 @@ retry_options = { } ``` -#### Automatically handle the `Retry-After` header +#### Automatically handle the `Retry-After` and `RateLimit-Reset` headers -Some APIs, like the [Slack API](https://api.slack.com/docs/rate-limits), will inform you when you reach their API limits by replying with a response status code of `429` and a response header of `Retry-After` containing a time in seconds. You should then only retry querying after the amount of time provided by the `Retry-After` header, otherwise you won't get a response. +Some APIs, like the [Slack API](https://api.slack.com/docs/rate-limits), will inform you when you reach their API limits by replying with a response status code of `429` +and a response header of `Retry-After` containing a time in seconds. You should then only retry querying after the amount of time provided by the `Retry-After` header, +otherwise you won't get a response. Other APIs communicate their rate limits via the [RateLimit-xxx](https://tools.ietf.org/id/draft-polli-ratelimit-headers-00.html#rfc.section.3.3) headers +where `RateLimit-Reset` behaves similarly to the `Retry-After`. -You can automatically handle this and have Faraday pause and retry for the right amount of time by including the `429` status code in the retry statuses list: +You can automatically handle both headers and have Faraday pause and retry for the right amount of time by including the `429` status code in the retry statuses list: ```ruby retry_options = {