node-fetchをやめる (#2493)

package.json

@@ -111,7 +111,6 @@
 		"ms": "2.1.3",
 		"multer": "1.4.5-lts.1",
 		"nested-property": "4.0.0",
-		"node-fetch": "2.6.12",
 		"nodemailer": "6.9.7",
 		"nprogress": "0.2.0",
 		"oauth": "0.10.0",
@@ -224,7 +223,6 @@
 		"@types/lolex": "5.1.6",
 		"@types/mocha": "10.0.6",
 		"@types/node": "20.10.0",
-		"@types/node-fetch": "2.6.2",
 		"@types/nodemailer": "6.4.14",
 		"@types/nprogress": "0.2.3",
 		"@types/oauth": "0.9.4",

src/misc/captcha.ts

@@ -1,6 +1,5 @@
-import fetch from 'node-fetch';
 import { URLSearchParams } from 'url';
-import { getAgentByUrl } from './fetch';
+import { getResponse } from './fetch';
 import config from '../config';
 
 export async function verifyRecaptcha(secret: string, response: string) {
@@ -36,21 +35,18 @@ async function getCaptchaResponse(url: string, secret: string, response: string)
 		response
 	});
 
-	const res = await fetch(url, {
+	const res = await getResponse({
+		url,
 		method: 'POST',
-		body: params,
+		body: params.toString(),
 		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded',
 			'User-Agent': config.userAgent
 		},
 		timeout: 10 * 1000,
-		agent: getAgentByUrl
 	}).catch(e => {
 		throw `${e.message || e}`;
 	});
 
-	if (!res.ok) {
-		throw `${res.status}`;
-	}
-
-	return await res.json() as CaptchaResponse;
+	return await JSON.parse(res.body) as CaptchaResponse;
 }

src/misc/check-private-ip.ts

@@ -0,0 +1,19 @@
+import config from '../config';
+import * as IPCIDR from 'ip-cidr';
+const PrivateIp = require('private-ip');
+
+export function checkPrivateIp(ip: string | undefined): boolean {
+	if ((process.env.NODE_ENV === 'production') && !config.proxy && ip) {
+		// check exclusion
+		for (const net of config.allowedPrivateNetworks || []) {
+			const cidr = new IPCIDR(net);
+			if (cidr.contains(ip)) {
+				return false;
+			}
+		}
+
+		return PrivateIp(ip);
+	} else {
+		return false;
+	}
+}

src/misc/download-url.ts

@@ -4,11 +4,9 @@ import * as util from 'util';
 import got, * as Got from 'got';
 import { httpAgent, httpsAgent, StatusError } from './fetch';
 import config from '../config';
-import * as chalk from 'chalk';
 import Logger from '../services/logger';
-import * as IPCIDR from 'ip-cidr';
+import { checkPrivateIp } from './check-private-ip';
 import { checkAllowedUrl } from './check-allowed-url';
-const PrivateIp = require('private-ip');
 
 const pipeline = util.promisify(stream.pipeline);
 
@@ -17,9 +15,9 @@ export async function downloadUrl(url: string, path: string) {
 		throw new StatusError('Invalid URL', 400);
 	}
 
-	const logger = new Logger('download');
+	const logger = new Logger('download-url');
 
-	logger.info(`Downloading ${chalk.cyan(url)} ...`);
+	logger.info(`Downloading ${url} ...`);
 
 	const timeout = 30 * 1000;
 	const operationTimeout = 60 * 1000;
@@ -50,11 +48,9 @@ export async function downloadUrl(url: string, path: string) {
 			req.destroy();
 		}
 	}).on('response', (res: Got.Response) => {
-		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
-			if (isPrivateIp(res.ip)) {
-				logger.warn(`Blocked address: ${res.ip}`);
-				req.destroy();
-			}
+		if (checkPrivateIp(res.ip)) {
+			logger.warn(`Blocked address: ${res.ip}`);
+			req.destroy();
 		}
 
 		const contentLength = res.headers['content-length'];
@@ -66,7 +62,7 @@ export async function downloadUrl(url: string, path: string) {
 			}
 		}
 	}).on('downloadProgress', (progress: Got.Progress) => {
-		if (progress.transferred > maxSize) {
+		if (progress.transferred > maxSize && progress.percent !== 1) {
 			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
 			req.destroy();
 		}
@@ -82,16 +78,5 @@ export async function downloadUrl(url: string, path: string) {
 		}
 	}
 
-	logger.succ(`Download finished: ${chalk.cyan(url)}`);
-}
-
-function isPrivateIp(ip: string) {
-	for (const net of config.allowedPrivateNetworks || []) {
-		const cidr = new IPCIDR(net);
-		if (cidr.contains(ip)) {
-			return false;
-		}
-	}
-
-	return PrivateIp(ip);
+	logger.succ(`Download finished: ${url}`);
 }

src/misc/fetch.ts

@@ -1,16 +1,14 @@
 import * as http from 'http';
 import * as https from 'https';
-import fetch from 'node-fetch';
 import CacheableLookup from 'cacheable-lookup';
+import got, * as Got from 'got';
 import { HttpProxyAgent } from 'http-proxy-agent';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import config from '../config';
-import { AbortController } from 'abort-controller';
-import Logger from '../services/logger';
+import { checkPrivateIp } from './check-private-ip';
+import { checkAllowedUrl } from './check-allowed-url';
 
-const logger = new Logger('fetch');
-
-export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) {
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<any> {
 	const res = await getResponse({
 		url,
 		method: 'GET',
@@ -22,7 +20,9 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
 		size: 1024 * 256,
 	});
 
-	return await res.json();
+	if (res.body.length > 65536) throw new Error('too large JSON');
+
+	return await JSON.parse(res.body); 
 }
 
 export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> {
@@ -36,32 +36,89 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout =
 		timeout
 	});
 
-	return await res.text();
+	return await res.body;
 }
 
-export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
-	logger.debug(`${args.method.toUpperCase()} ${args.url}\nHeaders: ${JSON.stringify(args.headers, null, 2)}${args.body ? `\n${args.body}` : ''}`);
+const RESPONSE_TIMEOUT = 30 * 1000;
+const OPERATION_TIMEOUT = 60 * 1000;
+const MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
 
-	const timeout = args?.timeout || 10 * 1000;
+export async function getResponse(args: { url: string, method: 'GET' | 'POST', body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
+	if (!checkAllowedUrl(args.url)) {
+		throw new StatusError('Invalid URL', 400);
+	}
 
-	const controller = new AbortController();
-	setTimeout(() => {
-		controller.abort();
-	}, timeout * 6);
+	const timeout = args.timeout || RESPONSE_TIMEOUT;
+	const operationTimeout = args.timeout ? args.timeout * 6 : OPERATION_TIMEOUT;
 
-	const res = await fetch(args.url, {
+	const req = got<string>(args.url, {
 		method: args.method,
 		headers: args.headers,
 		body: args.body,
-		timeout,
-		size: args?.size || 10 * 1024 * 1024,
-		agent: getAgentByUrl,
-		signal: controller.signal,
+		timeout: {
+			lookup: timeout,
+			connect: timeout,
+			secureConnect: timeout,
+			socket: timeout,	// read timeout
+			response: timeout,
+			send: timeout,
+			request: operationTimeout,	// whole operation timeout
+		},
+		agent: {
+			http: httpAgent,
+			https: httpsAgent,
+		},
+		http2: false,
+		retry: 0,
 	});
 
-	if (!res.ok) {
-		throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
-	}
+	req.on('redirect', (res, opts) => {
+		if (!checkAllowedUrl(opts.url)) {
+			req.cancel(`Invalid url: ${opts.url}`);
+		}
+	});
+
+	return await receiveResponce(req, args.size || MAX_RESPONSE_SIZE);
+}
+
+/**
+ * Receive response (with size limit)
+ * @param req Request
+ * @param maxSize size limit
+ */
+async function receiveResponce<T>(req: Got.CancelableRequest<Got.Response<T>>, maxSize: number) {
+	req.on('response', (res: Got.Response) => {
+		if (checkPrivateIp(res.ip)) {
+			req.cancel(`Blocked address: ${res.ip}`);
+		}
+	});
+
+	// 応答ヘッダでサイズチェック
+	req.on('response', (res: Got.Response) => {
+		const contentLength = res.headers['content-length'];
+		if (contentLength != null) {
+			const size = Number(contentLength);
+			if (size > maxSize) {
+				req.cancel(`maxSize exceeded (${size} > ${maxSize}) on response`);
+			}
+		}
+	});
+
+	// 受信中のデータでサイズチェック
+	req.on('downloadProgress', (progress: Got.Progress) => {
+		if (progress.transferred > maxSize && progress.percent !== 1) {
+			req.cancel(`maxSize exceeded (${progress.transferred} > ${maxSize}) on response`);
+		}
+	});
+
+	// 応答取得 with ステータスコードエラーの整形
+	const res = await req.catch(e => {
+		if (e instanceof Got.HTTPError) {
+			throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
+		} else {
+			throw e;
+		}
+	});
 
 	return res;
 }
@@ -126,6 +183,7 @@ export function getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https
 		return url.protocol == 'http:' ? httpAgent : httpsAgent;
 	}
 }
+//#endregion Agent
 
 export class StatusError extends Error {
 	public statusCode: number;

src/remote/activitypub/request.ts

@@ -24,12 +24,15 @@ export default async (user: ILocalUser, url: string, object: any) => {
 		}
 	});
 
-	await getResponse({
+	const res = await getResponse({
 		url,
 		method: req.request.method,
 		headers: req.request.headers,
 		body,
+		timeout: 10 * 1000,
 	});
+
+	return `${res.statusCode} ${res.statusMessage} ${res.body}`;
 };
 
 /**
@@ -59,5 +62,7 @@ export async function signedGet(url: string, user: ILocalUser) {
 		headers: req.request.headers
 	});
 
-	return await res.json();
+	if (res.body.length > 65536) throw new Error('too large JSON');
+
+	return await JSON.parse(res.body);
 }

test/utils.ts

@@ -1,5 +1,4 @@
 import * as childProcess from 'child_process';
-import fetch from 'node-fetch';
 import * as http from 'http';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -101,16 +100,27 @@ export const api = async (endpoint: string, params: any, me?: any): Promise<{ bo
 		i: me.token
 	} : {};
 
-	const res = await fetch(`http://localhost:${port}/api/${endpoint}`, {
+	const res = await got<string>(`http://localhost:${port}/api/${endpoint}`, {
 		method: 'POST',
 		headers: {
 			'Content-Type': 'application/json'
 		},
-		body: JSON.stringify(Object.assign(auth, params))
+		body: JSON.stringify(Object.assign(auth, params)),
+		timeout: 30 * 1000,
+		retry: 0,
+		hooks: {
+			beforeError: [
+				error => {
+					const { response } = error;
+					if (response && response.body) console.warn(response.body);
+					return error;
+				}
+			]
+		},
 	});
 
-	const status = res.status;
-	const body = res.status !== 204 ? await res.json().catch() : null;
+	const status = res.statusCode;
+	const body = res.statusCode !== 204 ? await JSON.parse(res.body) : null;
 
 	return {
 		status,