[core] rate limit handling on online file source

cmake/core-files.cmake

@@ -421,6 +421,8 @@ set(MBGL_CORE_FILES
     src/mbgl/util/grid_index.hpp
     src/mbgl/util/http_header.cpp
     src/mbgl/util/http_header.hpp
+    src/mbgl/util/http_timeout.cpp
+    src/mbgl/util/http_timeout.hpp
     src/mbgl/util/interpolate.hpp
     src/mbgl/util/intersection_tests.cpp
     src/mbgl/util/intersection_tests.hpp

cmake/test-files.cmake

@@ -87,6 +87,7 @@ set(MBGL_TEST_FILES
     # util
     test/util/async_task.cpp
     test/util/geo.cpp
+    test/util/http_timeout.cpp
     test/util/image.cpp
     test/util/mapbox.cpp
     test/util/memory.cpp

include/mbgl/storage/response.hpp

@@ -2,6 +2,7 @@
 
 #include <mbgl/util/chrono.hpp>
 #include <mbgl/util/optional.hpp>
+#include <mbgl/util/variant.hpp>
 
 #include <string>
 #include <memory>
@@ -45,15 +46,18 @@ class Response::Error {
         NotFound = 2,
         Server = 3,
         Connection = 4,
+        RateLimit = 5,
         Other = 6,
     } reason = Reason::Other;
 
     // An error message from the request handler, e.g. a server message or a system message
     // informing the user about the reason for the failure.
     std::string message;
+
+    optional<Timestamp> retryAfter;
 
 public:
-    Error(Reason, std::string = "");
+    Error(Reason, std::string = "", optional<Timestamp> = {});
 };
 
 std::ostream& operator<<(std::ostream&, Response::Error::Reason);

include/mbgl/util/chrono.hpp

@@ -29,6 +29,8 @@ std::string rfc1123(Timestamp);
 std::string iso8601(Timestamp);
 
 Timestamp parseTimestamp(const char *);
+
+Timestamp parseTimestamp(const int32_t timestamp);
 
 // C++17 polyfill
 template <class Rep, class Period, class = std::enable_if_t<

include/mbgl/util/constants.hpp

@@ -42,6 +42,8 @@ constexpr Duration DEFAULT_FADE_DURATION = Milliseconds(300);
 constexpr Seconds CLOCK_SKEW_RETRY_TIMEOUT { 30 };
 
 constexpr UnitBezier DEFAULT_TRANSITION_EASE = { 0, 0, 0.25, 1 };
+
+constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5;
 
 } // namespace util
 

platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java

@@ -43,7 +43,7 @@ class HTTPRequest implements Callback {
 
     private native void nativeOnFailure(int type, String message);
 
-    private native void nativeOnResponse(int code, String etag, String modified, String cacheControl, String expires, byte[] body);
+    private native void nativeOnResponse(int code, String etag, String modified, String cacheControl, String expires, String retryAfter, String xRateLimitReset, byte[] body);
 
     private HTTPRequest(long nativePtr, String resourceUrl, String userAgent, String etag, String modified) {
         mNativePtr = nativePtr;
@@ -120,7 +120,14 @@ public void onResponse(Call call, Response response) throws IOException {
 
         mLock.lock();
         if (mNativePtr != 0) {
-            nativeOnResponse(response.code(), response.header("ETag"), response.header("Last-Modified"), response.header("Cache-Control"), response.header("Expires"), body);
+            nativeOnResponse(response.code(),
+                    response.header("ETag"),
+                    response.header("Last-Modified"),
+                    response.header("Cache-Control"),
+                    response.header("Expires"),
+                    response.header("Retry-After"),
+                    response.header("x-rate-limit-reset"),
+                    body);
         }
         mLock.unlock();
     }

platform/android/src/http_file_source.cpp

@@ -29,6 +29,7 @@ class HTTPRequest : public AsyncRequest {
     void onResponse(jni::JNIEnv&, int code,
                     jni::String etag, jni::String modified,
                     jni::String cacheControl, jni::String expires,
+                    jni::String retryAfter, jni::String xRateLimitReset,
                     jni::Array<jni::jbyte> body);
 
     static jni::Class<HTTPRequest> javaClass;
@@ -101,8 +102,11 @@ HTTPRequest::~HTTPRequest() {
 }
 
 void HTTPRequest::onResponse(jni::JNIEnv& env, int code,
-                             jni::String etag, jni::String modified, jni::String cacheControl,
-                             jni::String expires, jni::Array<jni::jbyte> body) {
+                             jni::String etag, jni::String modified,
+                             jni::String cacheControl, jni::String expires,
+                             jni::String jRetryAfter, jni::String jXRateLimitReset,
+                             jni::Array<jni::jbyte> body) {
+
     using Error = Response::Error;
 
     if (etag) {
@@ -135,6 +139,16 @@ void HTTPRequest::onResponse(jni::JNIEnv& env, int code,
         response.notModified = true;
     } else if (code == 404) {
         response.error = std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
+    } else if (code == 429) {
+        optional<std::string> retryAfter;
+        optional<std::string> xRateLimitReset;
+        if (jRetryAfter) {
+            retryAfter = jni::Make<std::string>(env, jRetryAfter);
+        }
+        if (jXRateLimitReset) {
+            xRateLimitReset = jni::Make<std::string>(env, jXRateLimitReset);
+        }
+        response.error = std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429", http::parseRetryHeaders(retryAfter, xRateLimitReset));
     } else if (code >= 500 && code < 600) {
         response.error = std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } + std::to_string(code));
     } else {

platform/android/src/jni.cpp

@@ -1508,6 +1508,9 @@ void setOfflineRegionObserver(JNIEnv *env, jni::jobject* offlineRegion_, jni::jo
                 case mbgl::Response::Error::Reason::Connection:
                     errorReason = "REASON_CONNECTION";
                     break;
+                case mbgl::Response::Error::Reason::RateLimit:
+                    errorReason = "REASON_RATE_LIMIT";
+                    break;
                 case mbgl::Response::Error::Reason::Other:
                     errorReason = "REASON_OTHER";
                     break;

platform/darwin/src/http_file_source.mm

@@ -10,6 +10,7 @@
 #import <Foundation/Foundation.h>
 
 #include <mutex>
+#include <chrono>
 
 @interface MBGLBundleCanary : NSObject
 @end
@@ -293,6 +294,22 @@ void cancel() {
                     } else if (responseCode == 404) {
                         response.error =
                             std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
+                    } else if (responseCode == 429) {
+                        //Get the standard header
+                        optional<std::string> retryAfter;
+                        NSString *retryAfterHeader = headers[@"Retry-After"];
+                        if (retryAfterHeader) {
+                            retryAfter = std::string([retryAfterHeader UTF8String]);
+                        }
+
+                        //Fallback mapbox specific header
+                        optional<std::string> xRateLimitReset;
+                        NSString *xReset = headers[@"x-rate-limit-reset"];
+                        if (xReset) {
+                            xRateLimitReset = std::string([xReset UTF8String]);
+                        }
+
+                        response.error = std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429", http::parseRetryHeaders(retryAfter, xRateLimitReset));
                     } else if (responseCode >= 500 && responseCode < 600) {
                         response.error =
                             std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } +

platform/default/http_file_source.cpp

@@ -4,6 +4,7 @@
 #include <mbgl/platform/log.hpp>
 
 #include <mbgl/util/util.hpp>
+#include <mbgl/util/optional.hpp>
 #include <mbgl/util/run_loop.hpp>
 #include <mbgl/util/string.hpp>
 #include <mbgl/util/timer.hpp>
@@ -80,6 +81,9 @@ class HTTPRequest : public AsyncRequest {
     std::shared_ptr<std::string> data;
     std::unique_ptr<Response> response;
 
+    optional<std::string> retryAfter;
+    optional<std::string> xRateLimitReset;
+
     CURL *handle = nullptr;
     curl_slist *headers = nullptr;
 
@@ -325,6 +329,10 @@ size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const
     } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) {
         const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
         baton->response->expires = Timestamp{ Seconds(curl_getdate(value.c_str(), nullptr)) };
+    } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) {
+        baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n
+    } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) {
+        baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n
     }
 
     return length;
@@ -372,6 +380,10 @@ void HTTPRequest::handleResult(CURLcode code) {
         } else if (responseCode == 404) {
             response->error =
                 std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
+        } else if (responseCode == 429) {
+            response->error =
+                std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429",
+                                        http::parseRetryHeaders(retryAfter, xRateLimitReset));
         } else if (responseCode >= 500 && responseCode < 600) {
             response->error =
                 std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } +

platform/default/online_file_source.cpp

@@ -13,6 +13,7 @@
 #include <mbgl/util/async_task.hpp>
 #include <mbgl/util/noncopyable.hpp>
 #include <mbgl/util/timer.hpp>
+#include <mbgl/util/http_timeout.hpp>
 
 #include <algorithm>
 #include <cassert>
@@ -48,6 +49,7 @@ class OnlineFileRequest : public AsyncRequest {
     // backoff when retrying requests.
     uint32_t failedRequests = 0;
     Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
+    optional<Timestamp> retryAfter;
 };
 
 class OnlineFileSource::Impl {
@@ -200,30 +202,6 @@ OnlineFileRequest::~OnlineFileRequest() {
     impl.remove(this);
 }
 
-static Duration errorRetryTimeout(Response::Error::Reason failedRequestReason, uint32_t failedRequests) {
-    if (failedRequestReason == Response::Error::Reason::Server) {
-        // Retry after one second three times, then start exponential backoff.
-        return Seconds(failedRequests <= 3 ? 1 : 1 << std::min(failedRequests - 3, 31u));
-    } else if (failedRequestReason == Response::Error::Reason::Connection) {
-        // Immediate exponential backoff.
-        assert(failedRequests > 0);
-        return Seconds(1 << std::min(failedRequests - 1, 31u));
-    } else {
-        // No error, or not an error that triggers retries.
-        return Duration::max();
-    }
-}
-
-static Duration expirationTimeout(optional<Timestamp> expires, uint32_t expiredRequests) {
-    if (expiredRequests) {
-        return Seconds(1 << std::min(expiredRequests - 1, 31u));
-    } else if (expires) {
-        return std::max(Seconds::zero(), *expires - util::now());
-    } else {
-        return Duration::max();
-    }
-}
-
 Timestamp interpolateExpiration(const Timestamp& current,
                                 optional<Timestamp> prior,
                                 bool& expired) {
@@ -267,8 +245,9 @@ void OnlineFileRequest::schedule(optional<Timestamp> expires) {
 
     // If we're not being asked for a forced refresh, calculate a timeout that depends on how many
     // consecutive errors we've encountered, and on the expiration time, if present.
-    Duration timeout = std::min(errorRetryTimeout(failedRequestReason, failedRequests),
-        expirationTimeout(expires, expiredRequests));
+    Duration timeout = std::min(
+                            http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
+                            http::expirationTimeout(expires, expiredRequests));
 
     if (timeout == Duration::max()) {
         return;
@@ -321,6 +300,7 @@ void OnlineFileRequest::completed(Response response) {
     if (response.error) {
         failedRequests++;
         failedRequestReason = response.error->reason;
+        retryAfter = response.error->retryAfter;
     } else {
         failedRequests = 0;
         failedRequestReason = Response::Error::Reason::Success;

platform/qt/src/http_request.cpp

@@ -3,6 +3,7 @@
 
 #include <mbgl/storage/response.hpp>
 #include <mbgl/util/chrono.hpp>
+#include <mbgl/util/optional.hpp>
 #include <mbgl/util/http_header.hpp>
 #include <mbgl/util/string.hpp>
 
@@ -68,6 +69,8 @@ void HTTPRequest::handleNetworkReply(QNetworkReply *reply)
     }
 
     QPair<QByteArray, QByteArray> line;
+    optional<std::string> retryAfter;
+    optional<std::string> xRateLimitReset;
     foreach(line, reply->rawHeaderPairs()) {
         QString header = QString(line.first).toLower();
 
@@ -79,6 +82,10 @@ void HTTPRequest::handleNetworkReply(QNetworkReply *reply)
             response.expires = http::CacheControl::parse(line.second.constData()).toTimePoint();
         } else if (header == "expires") {
             response.expires = util::parseTimestamp(line.second.constData());
+        } else if (header == "retry-after") {
+            retryAfter = std::string(line.second.constData(), line.second.size());
+        } else if (header == "x-rate-limit-reset") {
+            xRateLimitReset = std::string(line.second.constData(), line.second.size());
         }
     }
 
@@ -109,6 +116,11 @@ void HTTPRequest::handleNetworkReply(QNetworkReply *reply)
         }
         break;
     }
+    case 429:
+        response.error = std::make_unique<Error>(
+                Error::Reason::RateLimit, "HTTP status code 429", 
+                http::parseRetryHeaders(retryAfter, xRateLimitReset));
+        break;
     default:
         Response::Error::Reason reason = (responseCode >= 500 && responseCode < 600) ?
             Error::Reason::Server : Error::Reason::Other;

src/mbgl/storage/response.cpp

@@ -21,8 +21,8 @@ Response& Response::operator=(const Response& res) {
     return *this;
 }
 
-Response::Error::Error(Reason reason_, std::string message_)
-    : reason(reason_), message(std::move(message_)) {
+Response::Error::Error(Reason reason_, std::string message_, optional<Timestamp> retryAfter_)
+    : reason(reason_), message(std::move(message_)), retryAfter(std::move(retryAfter_)) {
 }
 
 std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
@@ -35,6 +35,8 @@ std::ostream& operator<<(std::ostream& os, Response::Error::Reason r) {
         return os << "Response::Error::Reason::Server";
     case Response::Error::Reason::Connection:
         return os << "Response::Error::Reason::Connection";
+    case Response::Error::Reason::RateLimit:
+        return os << "Response::Error::Reason::RateLimit";
     case Response::Error::Reason::Other:
         return os << "Response::Error::Reason::Other";
     }

src/mbgl/util/chrono.cpp

@@ -33,6 +33,10 @@ std::string iso8601(Timestamp timestamp) {
 Timestamp parseTimestamp(const char* timestamp) {
     return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(parse_date(timestamp)));
 }
+
+Timestamp parseTimestamp(const int32_t timestamp) {
+    return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::from_time_t(timestamp));
+}
 
 } // namespace util
 

src/mbgl/util/http_header.cpp

@@ -28,6 +28,26 @@ CacheControl CacheControl::parse(const std::string& value) {
 optional<Timestamp> CacheControl::toTimePoint() const {
     return maxAge ? util::now() + Seconds(*maxAge) : optional<Timestamp>{};
 }
+
+optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
+                                      const optional<std::string>& xRateLimitReset) {
+    if (retryAfter) {
+        try {
+            auto secs = std::chrono::seconds(std::stoi(*retryAfter));
+            return std::chrono::time_point_cast<Seconds>(std::chrono::system_clock::now() + secs);
+        } catch (...) {
+            return util::parseTimestamp((*retryAfter).c_str());
+        }
+    } else if (xRateLimitReset) {
+        try {
+            return util::parseTimestamp(std::stoi(*xRateLimitReset));
+        } catch (...) {
+            return {};
+        }
+    }
+
+    return {};
+}
 
 } // namespace http
 } // namespace mbgl

src/mbgl/util/http_header.hpp

@@ -17,6 +17,9 @@ class CacheControl {
 
     optional<Timestamp> toTimePoint() const;
 };
+
+optional<Timestamp> parseRetryHeaders(const optional<std::string>& retryAfter,
+                                      const optional<std::string>& xRateLimitReset);
 
 } // namespace http
 } // namespace mbgl