postinstall: implement retries

When fetching things during postinstall, actually retry downloads if they fail partway through. If nodejs/node/issues/43187 gets resolved, we may be able to switch to importing `node:undici` instead of adding another copy with possibly incompatible API. Signed-off-by: Mark Yen <mark.yen@suse.com>
mook-as · Sep 11, 2024 · 2cd81c4 · 2cd81c4
1 parent 8b18041
commit 2cd81c4
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 39 deletions.
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -817,6 +817,7 @@ udhcpc
 UEFI
 Unauthed
 UNCONFIGURED
+undici
 unexpose
 unexposing
 unfetch

diff --git a/package.json b/package.json
@@ -168,6 +168,7 @@
     "ts-node": "10.9.2",
     "tsconfig-paths": "4.2.0",
     "typescript": "5.5.4",
+    "undici": "^6.19.8",
     "vue-eslint-parser": "9.4.3",
     "vue-jest": "3.0.7",
     "vue-template-compiler": "2.7.16",

diff --git a/scripts/lib/download.ts b/scripts/lib/download.ts
@@ -9,6 +9,8 @@ import os from 'os';
 import path from 'path';
 import stream from 'stream';
 
+import { Agent, RetryAgent } from 'undici';
+
 import { simpleSpawn } from 'scripts/simple_process';
 
 type ChecksumAlgorithm = 'sha1' | 'sha256' | 'sha512';
@@ -30,18 +32,67 @@ export type ArchiveDownloadOptions = DownloadOptions & {
 };
 
 async function fetchWithRetry(url: string) {
-  while (true) {
-    try {
-      return await fetch(url, { redirect: 'follow' });
-    } catch (ex: any) {
-      if (ex && ex.errno === 'EAI_AGAIN') {
-        console.log(`Recoverable error downloading ${ url }, retrying...`);
-        continue;
+  const agent = new RetryAgent(new Agent({ bodyTimeout: 1_000 }), {
+    maxTimeout: 5_000,
+    retry:      (rawError, { state, opts }, cb) => {
+      const err: Error & {
+        code?: string;
+        statusCode?: number;
+        headers?: Record<string, string>,
+      } = rawError;
+      const { maxRetries, maxTimeout } = opts.retryOptions ?? {};
+      // Unfortunately, the Undici API doesn't allow us to just call the default
+      // retry handler without a bunch of hacky gymnastics; so we'll have to
+      // mostly duplicate it instead.
+      const errors = ['EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'EHOSTDOWN', 'ENETDOWN', 'ENETUNREACH'];
+      const UndiciPrefix = 'UND_ERR_'; // spellcheck-ignore-line
+
+      errors.push(...['BODY_TIMEOUT', 'CONNECT_TIMEOUT', 'REQ_RETRY', 'SOCKET'].map(e => UndiciPrefix + e ));
+      if (err.code && !errors.includes(err.code)) {
+        cb(err); // Unexpected error code.
+
+        return null;
       }
-      console.dir(ex);
-      throw ex;
-    }
-  }
+
+      const statusesToRetry = [429, 500, 502, 503, 504];
+
+      if (err.statusCode && !statusesToRetry.includes(err.statusCode)) {
+        cb(err); // Unexpected status code.
+
+        return null;
+      }
+
+      if (state.counter > (maxRetries ?? 5)) {
+        cb(err); // Maximum retries reached.
+
+        return null;
+      }
+
+      // retryAfterHeader is number of seconds, or a date string.
+      const retryAfterHeader = err.headers?.['retry-after'] || '';
+      const retryAfter = (() => {
+        if (!retryAfterHeader) {
+          return 0;
+        }
+
+        return Number(retryAfterHeader) * 1_000 || (Date.parse(retryAfterHeader) - Date.now());
+      })();
+      const retryTimeout = (() => {
+        if (retryAfter > 0) {
+          return Math.max(retryAfter, maxTimeout ?? 30_000);
+        }
+
+        return 1_000; // Hard code one second; we have a ten minute total for the whole download.
+      })();
+
+      console.log(`Recoverable error ${ err } (${ err.code }) downloading ${ url }, retrying after ${ Math.floor(retryTimeout / 1_000) } seconds...`);
+      setTimeout(() => cb(null), retryTimeout);
+
+      return null;
+    },
+  });
+
+  return await fetch(url, { redirect: 'follow', dispatcher: agent } as any);
 }
 
 /**

diff --git a/yarn.lock b/yarn.lock
@@ -12259,16 +12259,7 @@ string-length@^5.0.1:
     char-regex "^2.0.0"
     strip-ansi "^7.0.1"
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-string-width@^2.1.1, string-width@^4, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2:
+"string-width-cjs@npm:string-width@^4.2.0", string-width@^2.1.1, string-width@^4, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -12346,7 +12337,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -12367,13 +12358,6 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45"
@@ -13026,6 +13010,11 @@ undici-types@~5.26.4:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
+undici@^6.19.8:
+  version "6.19.8"
+  resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.8.tgz#002d7c8a28f8cc3a44ff33c3d4be4d85e15d40e1"
+  integrity sha512-U8uCCl2x9TK3WANvmBavymRzxbfFYG+tAu+fgx3zxQy3qdagQqBLwJVrdyO1TBfUXvfKveMKJZhpvUYoOjM+4g==
+
 unfetch@4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/unfetch/-/unfetch-4.2.0.tgz#7e21b0ef7d363d8d9af0fb929a5555f6ef97a3be"
@@ -13702,7 +13691,7 @@ word-wrap@~1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -13728,15 +13717,6 @@ wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"