diff --git a/src/api.ts b/src/api.ts index 94a3365..f721e41 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,4 @@ -import { TwitterAuth } from './auth'; +import { TwitterAuth, TwitterGuestAuth } from './auth'; import { ApiError } from './errors'; import { updateCookieJar } from './requests'; import { Headers } from 'headers-polyfill'; @@ -51,11 +51,11 @@ export async function requestApi( auth: TwitterAuth, method: 'GET' | 'POST' = 'GET', ): Promise> { - const headers = new Headers(); - await auth.installTo(headers, url); - let res: Response; do { + const headers = new Headers(); + await auth.installTo(headers, url); + try { res = await auth.fetch(url, { method, @@ -75,6 +75,13 @@ export async function requestApi( await updateCookieJar(auth.cookieJar(), res.headers); if (res.status === 429) { + if (auth instanceof TwitterGuestAuth) { + // If we're using guest auth, we can just get a new token on the next request + auth.deleteToken(); + await auth.cookieJar().removeAllCookies(); + continue; + } + /* Known headers at this point: - x-rate-limit-limit: Maximum number of requests per time period? diff --git a/src/rate-limit.test.ts b/src/rate-limit.test.ts new file mode 100644 index 0000000..fcc3008 --- /dev/null +++ b/src/rate-limit.test.ts @@ -0,0 +1,42 @@ +import { Scraper } from './scraper'; + +test( + 'scraper does not get rate-limited with guest auth', + async () => { + const scraper = new Scraper(); + + performance.mark('rate-limit-test:benchmark-one:start'); + await scraper.getTweet('1585338303800578049'); + performance.mark('rate-limit-test:benchmark-one:end'); + + const initialTime = performance.measure( + 'rate-limit-test:benchmark-one:duration', + 'rate-limit-test:benchmark-one:start', + 'rate-limit-test:benchmark-one:end', + ); + + const getNextTweet = async () => { + performance.mark('rate-limit-test:benchmark:start'); + const tweet = await scraper.getTweet('1585338303800578049'); + performance.mark('rate-limit-test:benchmark:end'); + return tweet; + }; + + const tweets = await Promise.all( + new Array(1000).fill(undefined).map(() => getNextTweet()), + ); + + const finalTime = performance.measure( + 'rate-limit-test:benchmark:duration', + 'rate-limit-test:benchmark:start', + 'rate-limit-test:benchmark:end', + ); + + expect(finalTime.duration).toBeLessThanOrEqual(initialTime.duration); + + tweets.forEach((tweet) => { + expect(tweet).toBeTruthy(); + }); + }, + 1000 * 120, +);