From 277ee493df25ae799f3f1faae3f18aea33d4feae Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Sun, 26 Mar 2023 17:45:42 -0700 Subject: [PATCH] fix: use 'node-fetch' as Octokit's fetch In v6.1.0 Octokit started using Node.js's built-in `fetch`[^1] (provided by [undici][1]. Unfortunately, it is not 100% compatible with `node-fetch`, and notably it doesn't support `HTTP_PROXY` environment variables[^2]. This change switches `osv-offline` to explicitly use `node-fetch`. Closes #252 [1]: https://github.com/nodejs/undici [^1]: https://github.com/octokit/request.js/commit/d000a0ab58b6b60872190d26e4952d3e0a863499 [^2]: https://github.com/nodejs/undici/issues/1650 --- package-lock.json | 87 +++++++++++++++++++++++- packages/osv-offline/package.json | 3 +- packages/osv-offline/src/lib/download.ts | 5 +- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2becbc46..ad63d1a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3885,6 +3885,14 @@ "node": ">=8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/dateformat": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", @@ -5003,6 +5011,28 @@ "bser": "2.1.1" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -5109,6 +5139,17 @@ "is-callable": "^1.1.3" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -7588,6 +7629,24 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -12767,6 +12826,14 @@ "makeerror": "1.0.12" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -12982,7 +13049,8 @@ "adm-zip": "~0.5.10", "fs-extra": "^11.1.0", "got": "^11.8.6", - "luxon": "^3.3.0" + "luxon": "^3.3.0", + "node-fetch": "^2.6.9" }, "devDependencies": { "@types/adm-zip": "0.5.0", @@ -13091,6 +13159,23 @@ "graceful-fs": "^4.1.6" } }, + "packages/osv-offline/node_modules/node-fetch": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.1.tgz", + "integrity": "sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "packages/osv-offline/node_modules/universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", diff --git a/packages/osv-offline/package.json b/packages/osv-offline/package.json index b6a43b9a..e49ff53e 100644 --- a/packages/osv-offline/package.json +++ b/packages/osv-offline/package.json @@ -13,7 +13,8 @@ "adm-zip": "~0.5.10", "fs-extra": "^11.1.0", "got": "^11.8.6", - "luxon": "^3.3.0" + "luxon": "^3.3.0", + "node-fetch": "^2.6.9" }, "devDependencies": { "@types/adm-zip": "0.5.0", diff --git a/packages/osv-offline/src/lib/download.ts b/packages/osv-offline/src/lib/download.ts index fe4a752b..3aaeaff6 100644 --- a/packages/osv-offline/src/lib/download.ts +++ b/packages/osv-offline/src/lib/download.ts @@ -1,4 +1,5 @@ import fs from 'fs-extra'; +import fetch from 'node-fetch'; import { Octokit } from '@octokit/rest'; import got from 'got'; import { Stream } from 'stream'; @@ -34,9 +35,7 @@ export async function tryDownloadDb(): Promise { return true; } - const octokitOptions = process.env.GITHUB_COM_TOKEN - ? { auth: process.env.GITHUB_COM_TOKEN } - : undefined; + const octokitOptions = { auth: process.env.GITHUB_COM_TOKEN, request: { fetch } }; let latestRelease = null; try {