From 141290384309fc21469b9cf8d09fea91fa8ab210 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 8 Feb 2023 07:26:04 +0100 Subject: [PATCH 1/9] feat: alpha version of new twitter bridge --- bridges/TwitterBridge.php | 27 ++++---- lib/TwitterClient.php | 126 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 lib/TwitterClient.php diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 36f67defabb..ec23e446e11 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -219,25 +219,20 @@ public function collectData() $tweets = []; // Get authentication information - $this->getApiKey(); // Try to get all tweets switch ($this->queriedContext) { case 'By username': - $user = $this->makeApiCall('/1.1/users/show.json', ['screen_name' => $this->getInput('u')]); - if (!$user) { - returnServerError('Requested username can\'t be found.'); - } - - $params = [ - 'user_id' => $user->id_str, - 'tweet_mode' => 'extended' - ]; + $cache = new FileCache(); + $cache->setScope('twitter'); + $cache->setKey(['cache']); + $api = new TwitterClient($cache); - $data = $this->makeApiCall('/1.1/statuses/user_timeline.json', $params); + $data = $api->fetchUserTweets($this->getInput('u')); break; case 'By keyword or hashtag': + die('Not implemented'); $params = [ 'q' => urlencode($this->getInput('q')), 'tweet_mode' => 'extended', @@ -248,6 +243,7 @@ public function collectData() break; case 'By list': + die('Not implemented'); $params = [ 'slug' => strtolower($this->getInput('list')), 'owner_screen_name' => strtolower($this->getInput('user')), @@ -258,6 +254,7 @@ public function collectData() break; case 'By list ID': + die('Not implemented'); $params = [ 'list_id' => $this->getInput('listid'), 'tweet_mode' => 'extended', @@ -284,7 +281,7 @@ public function collectData() } // Filter out unwanted tweets - foreach ($data as $tweet) { + foreach ($data->tweets as $tweet) { // Filter out retweets to remove possible duplicates of original tweet switch ($this->queriedContext) { case 'By keyword or hashtag': @@ -333,9 +330,9 @@ public function collectData() $realtweet = $tweet->retweeted_status; } - $item['username'] = $realtweet->user->screen_name; - $item['fullname'] = $realtweet->user->name; - $item['avatar'] = $realtweet->user->profile_image_url_https; + $item['username'] = $data->user_info->legacy->screen_name; + $item['fullname'] = $data->user_info->legacy->name; + $item['avatar'] = $data->user_info->legacy->profile_image_url_https; $item['timestamp'] = $realtweet->created_at; $item['id'] = $realtweet->id_str; $item['uri'] = self::URI . $item['username'] . '/status/' . $item['id']; diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php new file mode 100644 index 00000000000..ddb52c03491 --- /dev/null +++ b/lib/TwitterClient.php @@ -0,0 +1,126 @@ +cache = $cache; + $this->authorization = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + $this->data = $cache->loadData() ?? []; + } + + public function fetchUserTweets(string $screenName) + { + $this->fetchGuestToken(); + $userInfo = $this->fetchUserInfoByScreenName($screenName); + $timeline = $this->fetchTimeline($userInfo->rest_id); + $instructions = $timeline->data->user->result->timeline_v2->timeline->instructions; + $instruction = $instructions[1]; + if ($instruction->type !== 'TimelineAddEntries') { + throw new \Exception('Unexpected instruction type'); + } + $tweets = []; + foreach ($instruction->entries as $entry) { + if ($entry->content->entryType !== 'TimelineTimelineItem') { + continue; + } + $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; + } + return (object)[ + 'user_info' => $userInfo, + 'tweets' => $tweets, + ]; + } + + private function fetchGuestToken(): void + { + if (isset($this->data['guest_token'])) { + return; + } + $url = 'https://api.twitter.com/1.1/guest/activate.json'; + $response = getContents($url, $this->createHttpHeaders(), [CURLOPT_POST => true]); + $this->data['guest_token'] = json_decode($response)->guest_token; + $this->cache->saveData($this->data); + } + + private function fetchUserInfoByScreenName(string $screenName) + { + if (isset($this->data[$screenName])) { + return $this->data[$screenName]; + } + $variables = [ + 'screen_name' => $screenName, + 'withHighlightedLabel' => true + ]; + $url = sprintf( + 'https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName?variables=%s', + urlencode(json_encode($variables)) + ); + $response = json_decode(getContents($url, $this->createHttpHeaders())); + $userInfo = $response->data->user; + $this->data[$screenName] = $userInfo; + $this->cache->saveData($this->data); + return $userInfo; + } + + private function fetchTimeline($userId) + { + $variables = [ + 'userId' => $userId, + 'count' => 40, + 'includePromotedContent' => true, + 'withQuickPromoteEligibilityTweetFields' => true, + 'withSuperFollowsUserFields' => true, + 'withDownvotePerspective' => false, + 'withReactionsMetadata' => false, + 'withReactionsPerspective' => false, + 'withSuperFollowsTweetFields' => true, + 'withVoice' => true, + 'withV2Timeline' => true, + ]; + $features = [ + 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, + 'responsive_web_graphql_exclude_directive_enabled' => false, + 'verified_phone_label_enabled' => false, + 'responsive_web_graphql_timeline_navigation_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'longform_notetweets_consumption_enabled' => true, + 'tweetypie_unmention_optimization_enabled' => true, + 'vibe_api_enabled' => true, + 'responsive_web_edit_tweet_api_enabled' => true, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => true, + 'view_counts_everywhere_api_enabled' => true, + 'freedom_of_speech_not_reach_appeal_label_enabled' => false, + 'standardized_nudges_misinfo' => true, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, + 'interactive_text_enabled' => true, + 'responsive_web_text_conversations_enabled' => false, + 'responsive_web_enhance_cards_enabled' => false, + ]; + $url = sprintf( + 'https://twitter.com/i/api/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets?variables=%s&features=%s', + urlencode(json_encode($variables)), + urlencode(json_encode($features)) + ); + $response = json_decode(getContents($url, $this->createHttpHeaders())); + return $response; + } + + private function createHttpHeaders(): array + { + $headers = [ + 'authorization' => sprintf('Bearer %s', $this->authorization), + 'x-guest-token' => $this->data['guest_token'] ?? null, + ]; + foreach ($headers as $key => $value) { + $headers[] = sprintf('%s: %s', $key, $value); + } + return $headers; + } +} From f51d4e69d792dd4423625ef58d91ea293a3d8221 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 8 Feb 2023 19:36:45 +0100 Subject: [PATCH 2/9] fix: refetch guest_token if expired --- lib/TwitterClient.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index ddb52c03491..b48ba9afd21 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -18,7 +18,16 @@ public function __construct(CacheInterface $cache) public function fetchUserTweets(string $screenName) { $this->fetchGuestToken(); - $userInfo = $this->fetchUserInfoByScreenName($screenName); + try { + $userInfo = $this->fetchUserInfoByScreenName($screenName); + } catch (HttpException $e) { + if ($e->getCode() === 403) { + // guest_token expired expired + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $userInfo = $this->fetchUserInfoByScreenName($screenName); + } + } $timeline = $this->fetchTimeline($userInfo->rest_id); $instructions = $timeline->data->user->result->timeline_v2->timeline->instructions; $instruction = $instructions[1]; From 8a383dce0640e3a96eb5002eac943bf15c21d2e7 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 8 Feb 2023 19:38:24 +0100 Subject: [PATCH 3/9] fix: purge cache --- bridges/TwitterBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index ec23e446e11..62cedf3cb52 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -226,6 +226,7 @@ public function collectData() $cache = new FileCache(); $cache->setScope('twitter'); $cache->setKey(['cache']); + $cache->purgeCache(60 * 60 * 3); // 3h $api = new TwitterClient($cache); $data = $api->fetchUserTweets($this->getInput('u')); From c68c91bf83f820c391d2d23800f90a73d6a7c067 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 8 Feb 2023 20:04:01 +0100 Subject: [PATCH 4/9] fix: safeguards --- lib/TwitterClient.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index b48ba9afd21..7b8b95bd429 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -29,10 +29,13 @@ public function fetchUserTweets(string $screenName) } } $timeline = $this->fetchTimeline($userInfo->rest_id); + if ($timeline->data->user->result->__typename === 'UserUnavailable') { + throw new \Exception('UserUnavailable'); + } $instructions = $timeline->data->user->result->timeline_v2->timeline->instructions; $instruction = $instructions[1]; if ($instruction->type !== 'TimelineAddEntries') { - throw new \Exception('Unexpected instruction type'); + throw new \Exception(sprintf('Unexpected instruction type: %s', $instruction->type)); } $tweets = []; foreach ($instruction->entries as $entry) { @@ -72,6 +75,9 @@ private function fetchUserInfoByScreenName(string $screenName) urlencode(json_encode($variables)) ); $response = json_decode(getContents($url, $this->createHttpHeaders())); + if ($response->errors) { + throw new Exception('Errors'); + } $userInfo = $response->data->user; $this->data[$screenName] = $userInfo; $this->cache->saveData($this->data); From 8e0483ded0f338c15aafe944f61411d7b062f400 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 9 Feb 2023 04:21:38 +0100 Subject: [PATCH 5/9] fix --- lib/TwitterClient.php | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 7b8b95bd429..05327ad2004 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -15,24 +15,32 @@ public function __construct(CacheInterface $cache) $this->data = $cache->loadData() ?? []; } - public function fetchUserTweets(string $screenName) + public function fetchUserTweets(string $screenName): \stdClass { $this->fetchGuestToken(); try { $userInfo = $this->fetchUserInfoByScreenName($screenName); } catch (HttpException $e) { if ($e->getCode() === 403) { + Logger::info('The guest token has expired'); // guest_token expired expired $this->data['guest_token'] = null; $this->fetchGuestToken(); $userInfo = $this->fetchUserInfoByScreenName($screenName); + } else { + throw $e; } } $timeline = $this->fetchTimeline($userInfo->rest_id); - if ($timeline->data->user->result->__typename === 'UserUnavailable') { + $result = $timeline->data->user->result; + if ($result->__typename === 'UserUnavailable') { throw new \Exception('UserUnavailable'); } - $instructions = $timeline->data->user->result->timeline_v2->timeline->instructions; + $instructionTypes = ['TimelineAddEntries', 'TimelineClearCache']; + $instructions = $result->timeline_v2->timeline->instructions; + if (!isset($instructions[1])) { + throw new \Exception('The account exists but has not tweeted yet?'); + } $instruction = $instructions[1]; if ($instruction->type !== 'TimelineAddEntries') { throw new \Exception(sprintf('Unexpected instruction type: %s', $instruction->type)); @@ -44,7 +52,7 @@ public function fetchUserTweets(string $screenName) } $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; } - return (object)[ + return (object) [ 'user_info' => $userInfo, 'tweets' => $tweets, ]; @@ -53,12 +61,15 @@ public function fetchUserTweets(string $screenName) private function fetchGuestToken(): void { if (isset($this->data['guest_token'])) { + Logger::info('Reusing cached guest token: ' . $this->data['guest_token']); return; } $url = 'https://api.twitter.com/1.1/guest/activate.json'; $response = getContents($url, $this->createHttpHeaders(), [CURLOPT_POST => true]); - $this->data['guest_token'] = json_decode($response)->guest_token; + $guest_token = json_decode($response)->guest_token; + $this->data['guest_token'] = $guest_token; $this->cache->saveData($this->data); + Logger::info("Fetch new guest token: $guest_token"); } private function fetchUserInfoByScreenName(string $screenName) @@ -75,8 +86,9 @@ private function fetchUserInfoByScreenName(string $screenName) urlencode(json_encode($variables)) ); $response = json_decode(getContents($url, $this->createHttpHeaders())); - if ($response->errors) { - throw new Exception('Errors'); + if (isset($response->errors)) { + // Grab the first error message + throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); } $userInfo = $response->data->user; $this->data[$screenName] = $userInfo; From 86fb519a8531851f864a93845ae5a480f1494533 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 9 Feb 2023 22:05:12 +0100 Subject: [PATCH 6/9] fix: two notices --- bridges/TwitterBridge.php | 3 +++ lib/TwitterClient.php | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 62cedf3cb52..a17c6cfba4e 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -283,6 +283,9 @@ public function collectData() // Filter out unwanted tweets foreach ($data->tweets as $tweet) { + if (!$tweet) { + continue; + } // Filter out retweets to remove possible duplicates of original tweet switch ($this->queriedContext) { case 'By keyword or hashtag': diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 05327ad2004..f9b5dfeca39 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -50,6 +50,9 @@ public function fetchUserTweets(string $screenName): \stdClass if ($entry->content->entryType !== 'TimelineTimelineItem') { continue; } + if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { + continue; + } $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; } return (object) [ From 77a0b6de317f9248738e5aed515f7882b7ee1af1 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 10 Feb 2023 09:21:38 +0100 Subject: [PATCH 7/9] fix --- lib/TwitterClient.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index f9b5dfeca39..fa8d765f837 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -23,7 +23,6 @@ public function fetchUserTweets(string $screenName): \stdClass } catch (HttpException $e) { if ($e->getCode() === 403) { Logger::info('The guest token has expired'); - // guest_token expired expired $this->data['guest_token'] = null; $this->fetchGuestToken(); $userInfo = $this->fetchUserInfoByScreenName($screenName); @@ -31,7 +30,20 @@ public function fetchUserTweets(string $screenName): \stdClass throw $e; } } - $timeline = $this->fetchTimeline($userInfo->rest_id); + + try { + $timeline = $this->fetchTimeline($userInfo->rest_id); + } catch (HttpException $e) { + if ($e->getCode() === 403) { + Logger::info('The guest token has expired'); + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $timeline = $this->fetchTimeline($userInfo->rest_id); + } else { + throw $e; + } + } + $result = $timeline->data->user->result; if ($result->__typename === 'UserUnavailable') { throw new \Exception('UserUnavailable'); From a6927e3794672d809cc6fb555d3db62efbeb2d1b Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 10 May 2023 20:02:02 +0200 Subject: [PATCH 8/9] fix: use factory to create cache --- bridges/TwitterBridge.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index a17c6cfba4e..e88979beeb7 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -223,7 +223,9 @@ public function collectData() // Try to get all tweets switch ($this->queriedContext) { case 'By username': - $cache = new FileCache(); + $cacheFactory = new CacheFactory(); + $cache = $cacheFactory->create(); + $cache->setScope('twitter'); $cache->setKey(['cache']); $cache->purgeCache(60 * 60 * 3); // 3h From 81fa80d63073237962b9767e103f3635018acc83 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 10 May 2023 20:19:35 +0200 Subject: [PATCH 9/9] fix: fail properly instead of die() --- bridges/TwitterBridge.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index e88979beeb7..31b97c86300 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -123,7 +123,7 @@ class TwitterBridge extends BridgeAbstract private $apiKey = null; private $guestToken = null; - private $authHeader = []; + private $authHeaders = []; public function detectParameters($url) { @@ -235,7 +235,7 @@ public function collectData() break; case 'By keyword or hashtag': - die('Not implemented'); + // Does not work with the recent twitter changes $params = [ 'q' => urlencode($this->getInput('q')), 'tweet_mode' => 'extended', @@ -246,7 +246,7 @@ public function collectData() break; case 'By list': - die('Not implemented'); + // Does not work with the recent twitter changes $params = [ 'slug' => strtolower($this->getInput('list')), 'owner_screen_name' => strtolower($this->getInput('user')), @@ -257,7 +257,7 @@ public function collectData() break; case 'By list ID': - die('Not implemented'); + // Does not work with the recent twitter changes $params = [ 'list_id' => $this->getInput('listid'), 'tweet_mode' => 'extended',