From afd0773e1ee9ca77b3eb8319bde2ea854fffe3dd Mon Sep 17 00:00:00 2001 From: arnds Date: Fri, 7 Jan 2022 16:27:17 +0100 Subject: [PATCH 1/5] Change getGuestToken to use Twitter-API V1.1 guest/activate for requesting new guest tokens Instead of searching inside base html page for the guest token, this patch instead uses the Twitter REST API V1.1 to aquire the nessecary guest tokens. --- bridges/TwitterBridge.php | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index a14e5062221..eba23990c8d 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -545,7 +545,7 @@ private function getApiKey() { $guestToken = null; if($guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2 || $guestTokenUses[0] <= 0 || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) { - $guestToken = $this->getGuestToken(); + $guestToken = $this->getGuestToken($apiKey); if ($guestToken === null) { if($guestTokenUses === null) { returnServerError('Could not parse guest token'); @@ -568,15 +568,17 @@ private function getApiKey() { // Get a guest token. This is different to an API key, // and it seems to change more regularly than the API key. - private function getGuestToken() { - $pageContent = getContents('https://twitter.com', array(), array(), true); - - $guestTokenRegex = '/gt=([0-9]*)/m'; - preg_match_all($guestTokenRegex, $pageContent['header'], $guestTokenMatches, PREG_SET_ORDER, 0); - if (!$guestTokenMatches) - preg_match_all($guestTokenRegex, $pageContent['content'], $guestTokenMatches, PREG_SET_ORDER, 0); - if (!$guestTokenMatches) return null; - $guestToken = $guestTokenMatches[0][1]; + private function getGuestToken($apiKey) { + $headers = array( + 'authorization: Bearer ' . $apiKey, + ); + $opts = array( + CURLOPT_POST => 1, + ); + + $pageContent = getContents('https://api.twitter.com/1.1/guest/activate.json', $headers, $opts, true); + $guestToken = json_decode($pageContent['content'])->guest_token; + return $guestToken; } From f866e75bf6eac9ee5d07b6ae7336203bcb78d973 Mon Sep 17 00:00:00 2001 From: root on skelington Date: Sat, 8 Jan 2022 17:54:18 +0200 Subject: [PATCH 2/5] Set retry-after on 503 Allow setting error 503 (temporary failure) and set a retry header for clients that understand 503 --- actions/DisplayAction.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 5db92b386a7..ad874b6ea3d 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -224,7 +224,11 @@ public function execute() { $items[] = $item; } elseif(Configuration::getConfig('error', 'output') === 'http') { - header('Content-Type: text/html', true, $this->get_return_code($e)); + $code = $this->get_return_code($e); + if ($code == 503) { + header('Retry-After: 600', true); + } + header('Content-Type: text/html', true, $code); die(buildTransformException($e, $bridge)); } } From e8305f9dcb39eeedced73bc4ace4fed79f66377c Mon Sep 17 00:00:00 2001 From: root on skelington Date: Sat, 8 Jan 2022 18:18:56 +0200 Subject: [PATCH 3/5] Add Twitter API retries Sometimes our cached guest token or API key is no longer valid, and Twitter returns 403. When this happens attempt to re-get an API key and get the contents. After API_RETRIES times, give up and return an error. --- bridges/TwitterBridge.php | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index eba23990c8d..a639e4424d5 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -5,6 +5,7 @@ class TwitterBridge extends BridgeAbstract { const API_URI = 'https://api.twitter.com'; const GUEST_TOKEN_USES = 100; const GUEST_TOKEN_EXPIRY = 10800; // 3hrs + const API_RETRIES = 3; const CACHE_TIMEOUT = 300; // 5min const DESCRIPTION = 'returns tweets'; const MAINTAINER = 'pmaziere'; @@ -481,7 +482,7 @@ private static function compareTweetId($tweet1, $tweet2) { //The aim of this function is to get an API key and a guest token //This function takes 2 requests, and therefore is cached - private function getApiKey() { + private function getApiKey($dumpToken) { $cacheFac = new CacheFactory(); $cacheFac->setWorkingDir(PATH_LIB_CACHES); @@ -506,7 +507,7 @@ private function getApiKey() { $data = $cache->loadData(); $apiKey = null; - if($data === null || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) { + if($dumpToken || $data === null || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) { $twitterPage = getContents('https://twitter.com'); $jsLink = false; @@ -543,7 +544,7 @@ private function getApiKey() { $guestTokenUses = $gt_cache->loadData(); $guestToken = null; - if($guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2 + if($dumpToken || $guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2 || $guestTokenUses[0] <= 0 || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) { $guestToken = $this->getGuestToken($apiKey); if ($guestToken === null) { @@ -583,11 +584,37 @@ private function getGuestToken($apiKey) { } private function getApiContents($uri) { - $apiKeys = $this->getApiKey(); - $headers = array('authorization: Bearer ' . $apiKeys[0], - 'x-guest-token: ' . $apiKeys[1], - ); - return getContents($uri, $headers); + $dump = false; + $lastCode = 500; + for ($i = 1; $i <= self::API_RETRIES; $i++) { + $apiKeys = $this->getApiKey($dump); + $headers = array('authorization: Bearer ' . $apiKeys[0], + 'x-guest-token: ' . $apiKeys[1], + ); + try { + $answer = getContents($uri, $headers); + return $answer; + } catch (Exception $e) { + // Sometimes Twitter invalidates our cached guest token, + // and returns 403. When this happens, dump the cached token, + // and retry. + $lastCode = $e->getCode(); + if ($lastCode == 403) { + $dump = true; + } else { + throw $e; + } + } + } + // If we reach here, either we never got a guestToken (500), or the token was invalid (403), + // so return a meaningful error. + if ($lastCode == 500) { + returnError("Could not obtain a guest token", 503); + } else if ($lastCode = 403) { + returnError("Our guest token is not valid", 503); + } else { + returnServerError("Unable to obtain contents"); + } } private function getRestId($username) { From f546a2b27a9bfe728b3bdb92dc20926c5973812a Mon Sep 17 00:00:00 2001 From: arnds Date: Mon, 10 Jan 2022 09:35:35 +0100 Subject: [PATCH 4/5] Catch http error, when (re-)aquirring a guesttoken fails. This enables to reuse old guesttokens until a new one was successfully aquired. --- bridges/TwitterBridge.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index eba23990c8d..1cc9d51d0af 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -576,9 +576,12 @@ private function getGuestToken($apiKey) { CURLOPT_POST => 1, ); - $pageContent = getContents('https://api.twitter.com/1.1/guest/activate.json', $headers, $opts, true); - $guestToken = json_decode($pageContent['content'])->guest_token; - + try { + $pageContent = getContents('https://api.twitter.com/1.1/guest/activate.json', $headers, $opts, true); + $guestToken = json_decode($pageContent['content'])->guest_token; + } catch (Exception $e) { + $guestToken = null; + } return $guestToken; } From e7d5e07c3427e6a54b97dcbfb99f15a3300f0299 Mon Sep 17 00:00:00 2001 From: root on skelington Date: Mon, 10 Jan 2022 15:47:33 +0200 Subject: [PATCH 5/5] Change double quotes to singles Change double quotes to singles to satisfy phpcs --- bridges/TwitterBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 2dac27c6dea..3fa00735fcf 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -612,11 +612,11 @@ private function getApiContents($uri) { // If we reach here, either we never got a guestToken (500), or the token was invalid (403), // so return a meaningful error. if ($lastCode == 500) { - returnError("Could not obtain a guest token", 503); + returnError('Could not obtain a guest token', 503); } else if ($lastCode = 403) { - returnError("Our guest token is not valid", 503); + returnError('Our guest token is not valid', 503); } else { - returnServerError("Unable to obtain contents"); + returnServerError('Unable to obtain contents'); } }