Exponential backoff and retry certain requests

lib/common/util.js

@@ -409,8 +409,39 @@ function makeWritableStream(dup, options, onComplete) {
 
 module.exports.makeWritableStream = makeWritableStream;
 
+/**
+ * Returns an exponential distributed time to wait given the number of retries
+ * that have been previously been attempted on the request.
+ *
+ * @param {number} retryNumber - The number of retries previously attempted.
+ * @return {number} An exponentially distributed time to wait E.g. for use with
+ * exponential backoff.
+ */
+function getNextRetryWait(retryNumber) {
+ return (Math.pow(2, retryNumber) * 1000) + Math.floor(Math.random() * 1000);
+}
+
+module.exports.getNextRetryWait = getNextRetryWait;
+
+/**
+ * Returns true if the API request should be retried, given the error that was
+ * given the first time the request was attempted. This is used for rate limit
+ * related errors as well as intermittent server errors.
+ *
+ * @param {error} err - The API error to check if it is appropriate to retry.
+ * @return {boolean} True if the API request should be retried, false otherwise.
+ */
+function shouldRetry(err) {
+ return !!err && [429, 500, 503].indexOf(err.code) !== -1;
+}
+
+module.exports.shouldRetryErr = shouldRetry;
+
 function makeAuthorizedRequest(config) {
  var GAE_OR_GCE = !config || (!config.credentials && !config.keyFile);
+ var MAX_RETRIES = config && config.maxRetries || 3;
+ var autoRetry = !config || config.autoRetry !== false ? true : false;
+ var attemptedRetries = 0;
 
  var missingCredentialsError = new Error();
  missingCredentialsError.message = [
@@ -475,12 +506,20 @@ function makeAuthorizedRequest(config) {
  return;
  }
 
+ function handleRateLimitResp(err, res, body) {
+ if (shouldRetry(err) && autoRetry && MAX_RETRIES > attemptedRetries) {
+ setTimeout(function() {
+ request(authorizedReqOpts, handleRateLimitResp);
+ }, getNextRetryWait(attemptedRetries++));
+ } else {
+ handleResp(err, res, body, callback);
+ }
+ }
+
  if (callback.onAuthorized) {
  callback.onAuthorized(null, authorizedReqOpts);
  } else {
- request(authorizedReqOpts, function(err, res, body) {
- handleResp(err, res, body, callback);
- });
+ request(authorizedReqOpts, handleRateLimitResp);
  }
  }
 

lib/index.js

@@ -82,6 +82,12 @@ var util = require('./common/util.js');
  * @param {object=} config.credentials - Credentials object.
  * @param {string} config.credentials.client_email
  * @param {string} config.credentials.private_key
+ * @param {boolean} config.autoRetry - Automatically retry requests if the
+ * response is related to rate limits or certain intermittent server errors.
+ * (default: true). Recommended is true. We will exponentially backoff
+ * subsequent requests by default.
+ * @param {number} config.maxRetries - Max number of auto retries to attempt
+ * before returning the error. (default: 3).
  *
  * @example
  * var gcloud = require('gcloud')({
@@ -102,6 +108,13 @@ var util = require('./common/util.js');
  * // properties may be overridden:
  * keyFilename: '/path/to/other/keyfile.json'
  * });
+ *
+ * @example
+ * var gcloud = require('gcloud')({
+ * keyFilename: '/path/to/keyfile.json',
+ * projectId: 'my-project',
+ * maxRetries: 5 // retry rate limited requests 5 times before giving up
+ * });
  */
 function gcloud(config) {
  return {

test/common/util.js

@@ -463,6 +463,190 @@ describe('common/util', function() {
  var makeRequest = util.makeAuthorizedRequest({});
  makeRequest({}, assert.ifError);
  });
+
+ it('should retry rate limit requests by default', function(done) {
+ var attemptedRetries = 0;
+ var error = new Error('Rate Limit Error.');
+ error.code = 429; // Rate limit error
+
+ var authorizedReqOpts = { a: 'b', c: 'd' };
+
+ var old_setTimeout = setTimeout;
+ setTimeout = function(callback, time) {
+ var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+ var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+ assert(time >= MIN_TIME && time <= MAX_TIME);
+ attemptedRetries++;
+ callback(); // make the request again
+ };
+
+ gsa_Override = function() {
+ return function authorize(reqOpts, callback) {
+ callback(null, authorizedReqOpts);
+ };
+ };
+
+ request_Override = function(reqOpts, callback) {
+ if (attemptedRetries === 3) {
+ setTimeout = old_setTimeout;
+ done();
+ } else {
+ callback(error); // this callback should check for rate limits
+ }
+ };
+
+ var makeRequest = util.makeAuthorizedRequest({});
+ makeRequest({}, assert.ifError);
+ });
+
+ it('should retry rate limits 3x on 429, 500, 503', function(done) {
+ var attemptedRetries = 0;
+ var codes = [429, 503, 500, 'done'];
+ var error = new Error('Rate Limit Error.');
+ error.code = codes[0]; // Rate limit error
+
+ var authorizedReqOpts = { a: 'b', c: 'd' };
+
+ var old_setTimeout = setTimeout;
+ setTimeout = function(callback, time) {
+ var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+ var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+ assert(time >= MIN_TIME && time <= MAX_TIME);
+ attemptedRetries++;
+ error.code = codes[attemptedRetries]; // test a new code
+ callback(); // make the request again
+ };
+
+ gsa_Override = function() {
+ return function authorize(reqOpts, callback) {
+ callback(null, authorizedReqOpts);
+ };
+ };
+
+ request_Override = function(reqOpts, callback) {
+ callback(error); // this callback should check for rate limits
+ };
+
+ var makeRequest = util.makeAuthorizedRequest({});
+ makeRequest({}, function(err) {
+ setTimeout = old_setTimeout;
+ assert.equal(err, error);
+ assert.equal(err.code, 'done');
+ done();
+ });
+ });
+
+ it('should retry rate limits 3x by default', function(done) {
+ var attemptedRetries = 0;
+ var error = new Error('Rate Limit Error.');
+ error.code = 429; // Rate limit error
+
+ var authorizedReqOpts = { a: 'b', c: 'd' };
+
+ var old_setTimeout = setTimeout;
+ setTimeout = function(callback, time) {
+ var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+ var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+ assert(time >= MIN_TIME && time <= MAX_TIME);
+ attemptedRetries++;
+ callback(); // make the request again
+ };
+
+ gsa_Override = function() {
+ return function authorize(reqOpts, callback) {
+ callback(null, authorizedReqOpts);
+ };
+ };
+
+ request_Override = function(reqOpts, callback) {
+ callback(error); // this callback should check for rate limits
+ };
+
+ var makeRequest = util.makeAuthorizedRequest({});
+ makeRequest({}, function(err) {
+ setTimeout = old_setTimeout;
+ assert.equal(attemptedRetries, 3);
+ assert.equal(err, error);
+ done();
+ });
+ });
+
+ it('should retry rate limits by maxRetries if provided', function(done) {
+ var MAX_RETRIES = 5;
+ var attemptedRetries = 0;
+ var error = new Error('Rate Limit Error.');
+ error.code = 429; // Rate limit error
+
+ var authorizedReqOpts = { a: 'b', c: 'd' };
+
+ var old_setTimeout = setTimeout;
+ setTimeout = function(callback, time) {
+ var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+ var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+ assert(time >= MIN_TIME && time <= MAX_TIME);
+ attemptedRetries++;
+ callback(); // make the request again
+ };
+
+ gsa_Override = function() {
+ return function authorize(reqOpts, callback) {
+ callback(null, authorizedReqOpts);
+ };
+ };
+
+ request_Override = function(reqOpts, callback) {
+ callback(error); // this callback should check for rate limits
+ };
+
+ var makeRequest = util.makeAuthorizedRequest({
+ maxRetries: MAX_RETRIES
+ });
+
+ makeRequest({}, function(err) {
+ setTimeout = old_setTimeout;
+ assert.equal(attemptedRetries, MAX_RETRIES);
+ assert.equal(err, error);
+ done();
+ });
+ });
+
+ it('should not retry rate limits if autoRetry is false', function(done) {
+ var attemptedRetries = 0;
+ var error = new Error('Rate Limit Error.');
+ error.code = 429; // Rate limit error
+
+ var authorizedReqOpts = { a: 'b', c: 'd' };
+
+ var old_setTimeout = setTimeout;
+ setTimeout = function(callback, time) {
+ var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000);
+ var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000;
+ assert(time >= MIN_TIME && time <= MAX_TIME);
+ attemptedRetries++;
+ callback(); // make the request again
+ };
+
+ gsa_Override = function() {
+ return function authorize(reqOpts, callback) {
+ callback(null, authorizedReqOpts);
+ };
+ };
+
+ request_Override = function(reqOpts, callback) {
+ callback(error); // this callback should check for rate limits
+ };
+
+ var makeRequest = util.makeAuthorizedRequest({
+ autoRetry: false
+ });
+
+ makeRequest({}, function(err) {
+ setTimeout = old_setTimeout;
+ assert.equal(attemptedRetries, 0);
+ assert.equal(err, error);
+ done();
+ });
+ });
  });
  });