From 7ab2f4b422b9b807958df2473a68aeb5946037ec Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 6 Jan 2022 16:04:35 +0100 Subject: [PATCH] fix: move queue from remote file node creation to remote file fetching --- packages/gatsby-core-utils/package.json | 1 + .../src/fetch-remote-file.ts | 58 +++++++++++++++++- .../gatsby-source-filesystem/package.json | 1 - .../src/create-remote-file-node.js | 61 ++----------------- 4 files changed, 62 insertions(+), 59 deletions(-) diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json index 86f079f9387b0..77aa86dee0dd9 100644 --- a/packages/gatsby-core-utils/package.json +++ b/packages/gatsby-core-utils/package.json @@ -32,6 +32,7 @@ "@babel/runtime": "^7.15.4", "ci-info": "2.0.0", "configstore": "^5.0.1", + "fastq": "^1.13.0", "file-type": "^16.5.3", "fs-extra": "^10.0.0", "got": "^11.8.3", diff --git a/packages/gatsby-core-utils/src/fetch-remote-file.ts b/packages/gatsby-core-utils/src/fetch-remote-file.ts index 52e98d20cfcba..1fa6b7e183e1a 100644 --- a/packages/gatsby-core-utils/src/fetch-remote-file.ts +++ b/packages/gatsby-core-utils/src/fetch-remote-file.ts @@ -10,6 +10,8 @@ import { } from "./filename-utils" import type { IncomingMessage } from "http" import type { GatsbyCache } from "gatsby" +import Queue from "fastq" +import type { queue, done } from "fastq" export interface IFetchRemoteFileOptions { url: string @@ -72,9 +74,63 @@ const ERROR_CODES_TO_RETRY = [ `ERR_GOT_REQUEST_ERROR`, ] +/******************** + * Queue Management * + ********************/ + +const GATSBY_CONCURRENT_DOWNLOAD = process.env.GATSBY_CONCURRENT_DOWNLOAD + ? parseInt(process.env.GATSBY_CONCURRENT_DOWNLOAD, 10) || 0 + : 200 + +const q: queue = Queue( + pushToQueue, + GATSBY_CONCURRENT_DOWNLOAD +) + +/** + * pushToQueue + * -- + * Handle tasks that are pushed in to the Queue + */ +async function pushToQueue( + task: IFetchRemoteFileOptions, + cb: done +): Promise { + try { + const node = await fetchFile(task) + return cb(null, node) + } catch (e) { + return cb(e) + } +} + +/** + * pushTask + * -- + * pushes a task in to the Queue and the processing cache + * + * Promisfy a task in queue + * @param {CreateRemoteFileNodePayload} task + * @return {Promise} + */ +async function pushTask(task: IFetchRemoteFileOptions): Promise { + return new Promise((resolve, reject) => { + q.push(task, (err, node) => { + if (!err) { + resolve(node) + } else { + reject(err) + } + }) + }) +} let fetchCache = new Map() let latestBuildId = `` +/*************************** + * Fetch remote file logic * + ***************************/ + export async function fetchRemoteFile( args: IFetchRemoteFileOptions ): Promise { @@ -91,7 +147,7 @@ export async function fetchRemoteFile( } // Create file fetch promise and store it into cache - const fetchPromise = fetchFile(args) + const fetchPromise = pushTask(args) fetchCache.set(args.url, fetchPromise) return fetchPromise.catch(err => { diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index e182bc1010f28..11c8ba2fe85bf 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -9,7 +9,6 @@ "dependencies": { "@babel/runtime": "^7.15.4", "chokidar": "^3.5.2", - "fastq": "^1.13.0", "file-type": "^16.5.3", "fs-extra": "^10.0.0", "gatsby-core-utils": "^3.5.0-next.3", diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 6f4596d1fc7f5..0a5abf6511486 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -6,7 +6,6 @@ const { } = require(`gatsby-core-utils`) const path = require(`path`) const { isWebUri } = require(`valid-url`) -const Queue = require(`fastq`) const { createFileNode } = require(`./create-file-node`) const { getRemoteFileExtension } = require(`./utils`) @@ -46,41 +45,6 @@ let showFlagWarning = !!process.env.GATSBY_EXPERIMENTAL_REMOTE_FILE_PLACEHOLDER * @param {Reporter} [options.reporter] */ -/******************** - * Queue Management * - ********************/ - -const GATSBY_CONCURRENT_DOWNLOAD = process.env.GATSBY_CONCURRENT_DOWNLOAD - ? parseInt(process.env.GATSBY_CONCURRENT_DOWNLOAD, 10) || 0 - : 200 - -const queue = Queue(pushToQueue, GATSBY_CONCURRENT_DOWNLOAD) - -/** - * @callback {Queue~queueCallback} - * @param {*} error - * @param {*} result - */ - -/** - * pushToQueue - * -- - * Handle tasks that are pushed in to the Queue - * - * - * @param {CreateRemoteFileNodePayload} task - * @param {Queue~queueCallback} cb - * @return {Promise} - */ -async function pushToQueue(task, cb) { - try { - const node = await processRemoteNode(task) - return cb(null, node) - } catch (e) { - return cb(e) - } -} - /****************** * Core Functions * ******************/ @@ -155,25 +119,6 @@ async function fetchPlaceholder({ fromPath, url, cache, ext, name }) { * Index of promises resolving to File node from remote url */ const processingCache = {} -/** - * pushTask - * -- - * pushes a task in to the Queue and the processing cache - * - * Promisfy a task in queue - * @param {CreateRemoteFileNodePayload} task - * @return {Promise} - */ -const pushTask = task => - new Promise((resolve, reject) => { - queue.push(task, (err, node) => { - if (!err) { - resolve(node) - } else { - reject(`failed to process ${task.url}\n${err}`) - } - }) - }) /*************** * Entry Point * @@ -245,11 +190,13 @@ module.exports = function createRemoteFileNode({ if (!url || isWebUri(url) === undefined) { return Promise.reject( - `url passed to createRemoteFileNode is either missing or not a proper web uri: ${url}` + new Error( + `url passed to createRemoteFileNode is either missing or not a proper web uri: ${url}` + ) ) } - const fileDownloadPromise = pushTask({ + const fileDownloadPromise = processRemoteNode({ url, cache, createNode,