feat: throttle requests on 429 (#12)

the-convocation · Jul 7, 2023 · 43686d2 · 43686d2
1 parent 4744540
commit 43686d2
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 15 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@the-convocation/twitter-scraper",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "main": "dist/_module.js",
   "repository": "https://github.com/the-convocation/twitter-scraper.git",
   "author": "karashiiro <49822414+karashiiro@users.noreply.github.com>",

diff --git a/src/api.ts b/src/api.ts
@@ -25,23 +25,29 @@ export async function requestApi<T>(
   await auth.installTo(headers, url);
 
   let res: Response;
-  try {
-    res = await fetch(url, {
-      method,
-      headers,
-    });
-  } catch (err) {
-    if (!(err instanceof Error)) {
-      throw err;
+  do {
+    try {
+      res = await fetch(url, {
+        method,
+        headers,
+      });
+    } catch (err) {
+      if (!(err instanceof Error)) {
+        throw err;
+      }
+
+      return {
+        success: false,
+        err: new Error('Failed to perform request.'),
+      };
     }
 
-    return {
-      success: false,
-      err: new Error('Failed to perform request.'),
-    };
-  }
+    await updateCookieJar(auth.cookieJar(), res.headers);
 
-  await updateCookieJar(auth.cookieJar(), res.headers);
+    if (res.status === 429) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  } while (res.status === 429);
 
   if (!res.ok) {
     return {