From af1af0a4679dcc07da368d13c6892a40a003e2ae Mon Sep 17 00:00:00 2001 From: Ryan Seys Date: Fri, 27 Mar 2015 23:09:13 -0400 Subject: [PATCH] Exponential backoff and retry certain requests --- lib/common/util.js | 45 ++++++++++- lib/index.js | 13 ++++ test/common/util.js | 184 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 3 deletions(-) diff --git a/lib/common/util.js b/lib/common/util.js index 1339fb12d71..762f39db469 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -409,8 +409,39 @@ function makeWritableStream(dup, options, onComplete) { module.exports.makeWritableStream = makeWritableStream; +/** + * Returns an exponential distributed time to wait given the number of retries + * that have been previously been attempted on the request. + * + * @param {number} retryNumber - The number of retries previously attempted. + * @return {number} An exponentially distributed time to wait E.g. for use with + * exponential backoff. + */ +function getNextRetryWait(retryNumber) { + return (Math.pow(2, retryNumber) * 1000) + Math.floor(Math.random() * 1000); +} + +module.exports.getNextRetryWait = getNextRetryWait; + +/** + * Returns true if the API request should be retried, given the error that was + * given the first time the request was attempted. This is used for rate limit + * related errors as well as intermittent server errors. + * + * @param {error} err - The API error to check if it is appropriate to retry. + * @return {boolean} True if the API request should be retried, false otherwise. + */ +function shouldRetry(err) { + return !!err && [429, 500, 503].indexOf(err.code) !== -1; +} + +module.exports.shouldRetryErr = shouldRetry; + function makeAuthorizedRequest(config) { var GAE_OR_GCE = !config || (!config.credentials && !config.keyFile); + var MAX_RETRIES = config && config.maxRetries || 3; + var autoRetry = !config || config.autoRetry !== false ? true : false; + var attemptedRetries = 0; var missingCredentialsError = new Error(); missingCredentialsError.message = [ @@ -475,12 +506,20 @@ function makeAuthorizedRequest(config) { return; } + function handleRateLimitResp(err, res, body) { + if (shouldRetry(err) && autoRetry && MAX_RETRIES > attemptedRetries) { + setTimeout(function() { + request(authorizedReqOpts, handleRateLimitResp); + }, getNextRetryWait(attemptedRetries++)); + } else { + handleResp(err, res, body, callback); + } + } + if (callback.onAuthorized) { callback.onAuthorized(null, authorizedReqOpts); } else { - request(authorizedReqOpts, function(err, res, body) { - handleResp(err, res, body, callback); - }); + request(authorizedReqOpts, handleRateLimitResp); } } diff --git a/lib/index.js b/lib/index.js index 1ae0d3c7a74..582af258c6f 100644 --- a/lib/index.js +++ b/lib/index.js @@ -82,6 +82,12 @@ var util = require('./common/util.js'); * @param {object=} config.credentials - Credentials object. * @param {string} config.credentials.client_email * @param {string} config.credentials.private_key + * @param {boolean} config.autoRetry - Automatically retry requests if the + * response is related to rate limits or certain intermittent server errors. + * (default: true). Recommended is true. We will exponentially backoff + * subsequent requests by default. + * @param {number} config.maxRetries - Max number of auto retries to attempt + * before returning the error. (default: 3). * * @example * var gcloud = require('gcloud')({ @@ -102,6 +108,13 @@ var util = require('./common/util.js'); * // properties may be overridden: * keyFilename: '/path/to/other/keyfile.json' * }); + * + * @example + * var gcloud = require('gcloud')({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project', + * maxRetries: 5 // retry rate limited requests 5 times before giving up + * }); */ function gcloud(config) { return { diff --git a/test/common/util.js b/test/common/util.js index 41f2531731d..e7c287f307a 100644 --- a/test/common/util.js +++ b/test/common/util.js @@ -463,6 +463,190 @@ describe('common/util', function() { var makeRequest = util.makeAuthorizedRequest({}); makeRequest({}, assert.ifError); }); + + it('should retry rate limit requests by default', function(done) { + var attemptedRetries = 0; + var error = new Error('Rate Limit Error.'); + error.code = 429; // Rate limit error + + var authorizedReqOpts = { a: 'b', c: 'd' }; + + var old_setTimeout = setTimeout; + setTimeout = function(callback, time) { + var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); + var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; + assert(time >= MIN_TIME && time <= MAX_TIME); + attemptedRetries++; + callback(); // make the request again + }; + + gsa_Override = function() { + return function authorize(reqOpts, callback) { + callback(null, authorizedReqOpts); + }; + }; + + request_Override = function(reqOpts, callback) { + if (attemptedRetries === 3) { + setTimeout = old_setTimeout; + done(); + } else { + callback(error); // this callback should check for rate limits + } + }; + + var makeRequest = util.makeAuthorizedRequest({}); + makeRequest({}, assert.ifError); + }); + + it('should retry rate limits 3x on 429, 500, 503', function(done) { + var attemptedRetries = 0; + var codes = [429, 503, 500, 'done']; + var error = new Error('Rate Limit Error.'); + error.code = codes[0]; // Rate limit error + + var authorizedReqOpts = { a: 'b', c: 'd' }; + + var old_setTimeout = setTimeout; + setTimeout = function(callback, time) { + var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); + var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; + assert(time >= MIN_TIME && time <= MAX_TIME); + attemptedRetries++; + error.code = codes[attemptedRetries]; // test a new code + callback(); // make the request again + }; + + gsa_Override = function() { + return function authorize(reqOpts, callback) { + callback(null, authorizedReqOpts); + }; + }; + + request_Override = function(reqOpts, callback) { + callback(error); // this callback should check for rate limits + }; + + var makeRequest = util.makeAuthorizedRequest({}); + makeRequest({}, function(err) { + setTimeout = old_setTimeout; + assert.equal(err, error); + assert.equal(err.code, 'done'); + done(); + }); + }); + + it('should retry rate limits 3x by default', function(done) { + var attemptedRetries = 0; + var error = new Error('Rate Limit Error.'); + error.code = 429; // Rate limit error + + var authorizedReqOpts = { a: 'b', c: 'd' }; + + var old_setTimeout = setTimeout; + setTimeout = function(callback, time) { + var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); + var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; + assert(time >= MIN_TIME && time <= MAX_TIME); + attemptedRetries++; + callback(); // make the request again + }; + + gsa_Override = function() { + return function authorize(reqOpts, callback) { + callback(null, authorizedReqOpts); + }; + }; + + request_Override = function(reqOpts, callback) { + callback(error); // this callback should check for rate limits + }; + + var makeRequest = util.makeAuthorizedRequest({}); + makeRequest({}, function(err) { + setTimeout = old_setTimeout; + assert.equal(attemptedRetries, 3); + assert.equal(err, error); + done(); + }); + }); + + it('should retry rate limits by maxRetries if provided', function(done) { + var MAX_RETRIES = 5; + var attemptedRetries = 0; + var error = new Error('Rate Limit Error.'); + error.code = 429; // Rate limit error + + var authorizedReqOpts = { a: 'b', c: 'd' }; + + var old_setTimeout = setTimeout; + setTimeout = function(callback, time) { + var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); + var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; + assert(time >= MIN_TIME && time <= MAX_TIME); + attemptedRetries++; + callback(); // make the request again + }; + + gsa_Override = function() { + return function authorize(reqOpts, callback) { + callback(null, authorizedReqOpts); + }; + }; + + request_Override = function(reqOpts, callback) { + callback(error); // this callback should check for rate limits + }; + + var makeRequest = util.makeAuthorizedRequest({ + maxRetries: MAX_RETRIES + }); + + makeRequest({}, function(err) { + setTimeout = old_setTimeout; + assert.equal(attemptedRetries, MAX_RETRIES); + assert.equal(err, error); + done(); + }); + }); + + it('should not retry rate limits if autoRetry is false', function(done) { + var attemptedRetries = 0; + var error = new Error('Rate Limit Error.'); + error.code = 429; // Rate limit error + + var authorizedReqOpts = { a: 'b', c: 'd' }; + + var old_setTimeout = setTimeout; + setTimeout = function(callback, time) { + var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); + var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; + assert(time >= MIN_TIME && time <= MAX_TIME); + attemptedRetries++; + callback(); // make the request again + }; + + gsa_Override = function() { + return function authorize(reqOpts, callback) { + callback(null, authorizedReqOpts); + }; + }; + + request_Override = function(reqOpts, callback) { + callback(error); // this callback should check for rate limits + }; + + var makeRequest = util.makeAuthorizedRequest({ + autoRetry: false + }); + + makeRequest({}, function(err) { + setTimeout = old_setTimeout; + assert.equal(attemptedRetries, 0); + assert.equal(err, error); + done(); + }); + }); }); });