feat: automatically recover from rate limit

gajus · Sep 16, 2017 · 627fefc · 627fefc
1 parent 9c1bb0a
commit 627fefc
Show file tree

Hide file tree

Showing 5 changed files with 162 additions and 58 deletions.
diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "ajv": "^5.2.2",
+    "bluefeather": "^2.7.0",
     "debug": "^3.0.1",
     "deep-map-keys": "^1.2.0",
     "es6-error": "^4.0.2",

diff --git a/src/Tmdb.js b/src/Tmdb.js
@@ -3,6 +3,9 @@
 import xfetch from 'xfetch';
 import qs from 'qs';
 import deepMapKeys from 'deep-map-keys';
+import {
+  delay
+} from 'bluefeather';
 import {
   camelCase
 } from 'lodash';
@@ -25,20 +28,6 @@ type QueryType = {
   [key: string]: string | number
 };
 
-const isResponseValid = async (intermediateResponse) => {
-  if (!String(intermediateResponse.status).startsWith('2') && !String(intermediateResponse.status).startsWith('3') && intermediateResponse.status !== 400) {
-    if (intermediateResponse.status === 404) {
-      throw new NotFoundError();
-    }
-
-    const response = await intermediateResponse.json();
-
-    throw new RemoteError(response.status_message, response.status_code);
-  }
-
-  return true;
-};
-
 class Tmdb {
   apiKey: string;
   language: string;
@@ -50,18 +39,57 @@ class Tmdb {
 
   // eslint-disable-next-line flowtype/no-weak-types
   async get (resource: string, parameters: QueryType = {}): Object {
-    const requestQuery = qs.stringify({
-      // eslint-disable-next-line id-match
-      api_key: this.apiKey,
-      ...parameters
-    });
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const requestQuery = qs.stringify({
+        // eslint-disable-next-line id-match
+        api_key: this.apiKey,
+        ...parameters
+      });
 
-    const body = await xfetch('https://api.themoviedb.org/3/' + resource + '?' + requestQuery, {
-      isResponseValid,
-      responseType: 'json'
-    });
+      const response = await xfetch('https://api.themoviedb.org/3/' + resource + '?' + requestQuery, {
+        isResponseValid: () => {
+          return true;
+        },
+        responseType: 'full'
+      });
+
+      if (!response.headers.has('x-ratelimit-remaining')) {
+        throw new UnexpectedResponseError();
+      }
 
-    return deepMapKeys(body, camelCase);
+      if (!String(response.status).startsWith('2')) {
+        const rateLimitRemaining = Number(response.headers.get('x-ratelimit-remaining'));
+
+        if (!rateLimitRemaining) {
+          const currentTime = Math.round(new Date().getTime() / 1000);
+          const rateLimitReset = Number(response.headers.get('x-ratelimit-reset'));
+
+          // The minimum 30 seconds cooldown ensures that in case 'x-ratelimit-reset'
+          // time is wrong, we don't bombard the TMDb server with requests.
+          const cooldownTime = Math.max(rateLimitReset - currentTime, 30);
+
+          debug('reached rate limit; waiting %d seconds', cooldownTime);
+
+          await delay(cooldownTime * 1000);
+
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        if (response.status === 404) {
+          throw new NotFoundError();
+        }
+
+        const errorBody = await response.json();
+
+        throw new RemoteError(errorBody.status_message, errorBody.status_code);
+      }
+
+      const body = await response.json();
+
+      return deepMapKeys(body, camelCase);
+    }
   }
 
   async findId (resourceType: 'movie', externalSource: 'imdb', externalId: string): Promise<number> {

diff --git a/test/Tmdb/findId.js b/test/Tmdb/findId.js
@@ -14,13 +14,19 @@ test('finds TMDb movie record ID using IMDb ID', async (t) => {
 
   const scope = nock('https://api.themoviedb.org')
     .get('/3/find/tt1?api_key=foo&external_source=imdb')
-    .reply(200, {
-      movie_results: [
-        {
-          id: 1
-        }
-      ]
-    });
+    .reply(
+      200,
+      {
+        movie_results: [
+          {
+            id: 1
+          }
+        ]
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const movieId = await tmdb.findId('movie', 'imdb', 'tt1');
 
@@ -35,9 +41,15 @@ test('throws NotFoundError if resource cannot be found', async (t) => {
 
   const scope = nock('https://api.themoviedb.org')
     .get('/3/find/tt1?api_key=foo&external_source=imdb')
-    .reply(200, {
-      movie_results: []
-    });
+    .reply(
+      200,
+      {
+        movie_results: []
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const error = await t.throws(tmdb.findId('movie', 'imdb', 'tt1'));
 
@@ -52,16 +64,22 @@ test('throws UnexpectedResponseError if multiple results are returned', async (t
 
   const scope = nock('https://api.themoviedb.org')
     .get('/3/find/tt1?api_key=foo&external_source=imdb')
-    .reply(200, {
-      movie_results: [
-        {
-          id: 1
-        },
-        {
-          id: 2
-        }
-      ]
-    });
+    .reply(
+      200,
+      {
+        movie_results: [
+          {
+            id: 1
+          },
+          {
+            id: 2
+          }
+        ]
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const error = await t.throws(tmdb.findId('movie', 'imdb', 'tt1'));
 

diff --git a/test/Tmdb/get.js b/test/Tmdb/get.js
@@ -14,23 +14,68 @@ test('creates a GET request using apiKey', async (t) => {
 
   const scope = nock('https://api.themoviedb.org')
     .get('/3/bar?api_key=foo')
-    .reply(200, {});
+    .reply(
+      200,
+      {},
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   await tmdb.get('bar');
 
   t.true(scope.isDone());
 });
 
+test('retries queries that have failed because of the rate limit', async (t) => {
+  const apiKey = 'foo';
+  const tmdb = new Tmdb(apiKey);
+
+  const currentTime = Math.round(new Date().getTime() / 1000);
+
+  const scope1 = nock('https://api.themoviedb.org')
+    .get('/3/bar?api_key=foo')
+    .reply(
+      429,
+      {},
+      {
+        'x-ratelimit-remaining': 0,
+        'x-ratelimit-reset': currentTime
+      }
+    );
+
+  const scope2 = nock('https://api.themoviedb.org')
+    .get('/3/bar?api_key=foo')
+    .reply(
+      200,
+      {},
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
+
+  await tmdb.get('bar');
+
+  t.true(scope1.isDone());
+  t.true(scope2.isDone());
+});
+
 test('throws NotFoundError if response is 404', async (t) => {
   const apiKey = 'foo';
   const tmdb = new Tmdb(apiKey);
 
   nock('https://api.themoviedb.org')
     .get('/3/bar?api_key=foo')
-    .reply(404, {
-      status_code: 34,
-      status_message: 'The resource you requested could not be found.'
-    });
+    .reply(
+      404,
+      {
+        status_code: 34,
+        status_message: 'The resource you requested could not be found.'
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const error = await t.throws(tmdb.get('bar'));
 
@@ -44,11 +89,17 @@ test('throws RemoteError if response is non-200', async (t) => {
 
   nock('https://api.themoviedb.org')
     .get('/3/bar?api_key=foo')
-    .reply(401, {
-      status_code: 7,
-      status_message: 'Invalid API key: You must be granted a valid key.',
-      success: false
-    });
+    .reply(
+      401,
+      {
+        status_code: 7,
+        status_message: 'Invalid API key: You must be granted a valid key.',
+        success: false
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const error = await t.throws(tmdb.get('bar'));
 

diff --git a/test/Tmdb/getMovie.js b/test/Tmdb/getMovie.js
@@ -10,10 +10,16 @@ test('retrieves movie resource', async (t) => {
 
   const scope = nock('https://api.themoviedb.org')
     .get('/3/movie/1?api_key=foo&language=en')
-    .reply(200, {
-      id: 1,
-      imdb_id: 'tt1'
-    });
+    .reply(
+      200,
+      {
+        id: 1,
+        imdb_id: 'tt1'
+      },
+      {
+        'x-ratelimit-remaining': 1
+      }
+    );
 
   const movie = await tmdb.getMovie(1);