add pause between requests to avoid a 503 response

ryancwalsh · Jan 11, 2020 · 5a07814 · 5a07814
1 parent 8ec7343
commit 5a07814
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 2 deletions.
diff --git a/composer.json b/composer.json
@@ -2,7 +2,7 @@
     "name": "ryancwalsh/stack-exchange-backup-laravel",
     "description": "My aim is to back up all of my questions and answers and anything else valuable in my accounts across all of the StackExchange sites (StackOverflow, SuperUser, https://apple.stackexchange.com/, https://askubuntu.com/, etc).",
     "type": "package",
-    "version": "2.0.3",
+    "version": "2.0.4",
     "require": {
         "guzzlehttp/guzzle": "^6.3",
         "php": "^7.3",

diff --git a/src/ExportStackExchangeHelper.php b/src/ExportStackExchangeHelper.php
@@ -15,6 +15,7 @@ class ExportStackExchangeHelper {
     const APP_FOLDER = 'app/';
     const SE_FOLDER = 'StackExchange/';
     const DOT_ZIP = '.zip';
+    const PAUSE_BETWEEN_REQUESTS_MS = 200;
 
     protected $client_id;
     protected $client_secret;
@@ -114,7 +115,17 @@ public function get($uri = '', array $options = [], $cacheMinutes = self::SESSIO
                     //Log::debug($fullUrl);
                     $httpClient = new HttpClient();
                     $response = $httpClient->request('get', $fullUrl);
-                    return $response->getBody()->getContents();
+                    $responseContentsJson = $response->getBody()->getContents();
+                    $responseArr = json_decode($responseContentsJson, true);
+                    Log::debug('quota_remaining=' . $responseArr['quota_remaining']); //https://stackapps.com/a/7832/55248
+                    $backoff = $responseArr['backoff'] ?? 0;
+                    if ($backoff) {
+                        Log::warning('Rate-limiting because backoff=' . $backoff . ' for ' . $uri);
+                        sleep($backoff); //ONEDAY: Figure out a better approach since `sleep` is highly discouraged. See https://github.com/ryancwalsh/StackExchangeBackupLaravelPHP/issues/11
+                    } else {
+                        usleep(self::PAUSE_BETWEEN_REQUESTS_MS * 1000); //preemptively rate-limiting the requests so that the API does not throw a 503 response. https://github.com/ryancwalsh/StackExchangeBackupLaravelPHP/issues/11
+                    }
+                    return $responseContentsJson;
                 });
         return $response;
     }