fix: move queue from remote file node creation to remote file fetching

gatsbyjs · Jan 6, 2022 · 7ab2f4b · 7ab2f4b
1 parent 2e32870
commit 7ab2f4b
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 59 deletions.
diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json
@@ -32,6 +32,7 @@
     "@babel/runtime": "^7.15.4",
     "ci-info": "2.0.0",
     "configstore": "^5.0.1",
+    "fastq": "^1.13.0",
     "file-type": "^16.5.3",
     "fs-extra": "^10.0.0",
     "got": "^11.8.3",

diff --git a/packages/gatsby-core-utils/src/fetch-remote-file.ts b/packages/gatsby-core-utils/src/fetch-remote-file.ts
@@ -10,6 +10,8 @@ import {
 } from "./filename-utils"
 import type { IncomingMessage } from "http"
 import type { GatsbyCache } from "gatsby"
+import Queue from "fastq"
+import type { queue, done } from "fastq"
 
 export interface IFetchRemoteFileOptions {
   url: string
@@ -72,9 +74,63 @@ const ERROR_CODES_TO_RETRY = [
   `ERR_GOT_REQUEST_ERROR`,
 ]
 
+/********************
+ * Queue Management *
+ ********************/
+
+const GATSBY_CONCURRENT_DOWNLOAD = process.env.GATSBY_CONCURRENT_DOWNLOAD
+  ? parseInt(process.env.GATSBY_CONCURRENT_DOWNLOAD, 10) || 0
+  : 200
+
+const q: queue<IFetchRemoteFileOptions> = Queue(
+  pushToQueue,
+  GATSBY_CONCURRENT_DOWNLOAD
+)
+
+/**
+ * pushToQueue
+ * --
+ * Handle tasks that are pushed in to the Queue
+ */
+async function pushToQueue(
+  task: IFetchRemoteFileOptions,
+  cb: done
+): Promise<unknown> {
+  try {
+    const node = await fetchFile(task)
+    return cb(null, node)
+  } catch (e) {
+    return cb(e)
+  }
+}
+
+/**
+ * pushTask
+ * --
+ * pushes a task in to the Queue and the processing cache
+ *
+ * Promisfy a task in queue
+ * @param {CreateRemoteFileNodePayload} task
+ * @return {Promise<Object>}
+ */
+async function pushTask(task: IFetchRemoteFileOptions): Promise<string> {
+  return new Promise((resolve, reject) => {
+    q.push(task, (err, node) => {
+      if (!err) {
+        resolve(node)
+      } else {
+        reject(err)
+      }
+    })
+  })
+}
 let fetchCache = new Map()
 let latestBuildId = ``
 
+/***************************
+ * Fetch remote file logic *
+ ***************************/
+
 export async function fetchRemoteFile(
   args: IFetchRemoteFileOptions
 ): Promise<string> {
@@ -91,7 +147,7 @@ export async function fetchRemoteFile(
   }
 
   // Create file fetch promise and store it into cache
-  const fetchPromise = fetchFile(args)
+  const fetchPromise = pushTask(args)
   fetchCache.set(args.url, fetchPromise)
 
   return fetchPromise.catch(err => {

diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json
@@ -9,7 +9,6 @@
   "dependencies": {
     "@babel/runtime": "^7.15.4",
     "chokidar": "^3.5.2",
-    "fastq": "^1.13.0",
     "file-type": "^16.5.3",
     "fs-extra": "^10.0.0",
     "gatsby-core-utils": "^3.5.0-next.3",

diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js
@@ -6,7 +6,6 @@ const {
 } = require(`gatsby-core-utils`)
 const path = require(`path`)
 const { isWebUri } = require(`valid-url`)
-const Queue = require(`fastq`)
 const { createFileNode } = require(`./create-file-node`)
 const { getRemoteFileExtension } = require(`./utils`)
 
@@ -46,41 +45,6 @@ let showFlagWarning = !!process.env.GATSBY_EXPERIMENTAL_REMOTE_FILE_PLACEHOLDER
  * @param  {Reporter} [options.reporter]
  */
 
-/********************
- * Queue Management *
- ********************/
-
-const GATSBY_CONCURRENT_DOWNLOAD = process.env.GATSBY_CONCURRENT_DOWNLOAD
-  ? parseInt(process.env.GATSBY_CONCURRENT_DOWNLOAD, 10) || 0
-  : 200
-
-const queue = Queue(pushToQueue, GATSBY_CONCURRENT_DOWNLOAD)
-
-/**
- * @callback {Queue~queueCallback}
- * @param {*} error
- * @param {*} result
- */
-
-/**
- * pushToQueue
- * --
- * Handle tasks that are pushed in to the Queue
- *
- *
- * @param  {CreateRemoteFileNodePayload}          task
- * @param  {Queue~queueCallback}  cb
- * @return {Promise<null>}
- */
-async function pushToQueue(task, cb) {
-  try {
-    const node = await processRemoteNode(task)
-    return cb(null, node)
-  } catch (e) {
-    return cb(e)
-  }
-}
-
 /******************
  * Core Functions *
  ******************/
@@ -155,25 +119,6 @@ async function fetchPlaceholder({ fromPath, url, cache, ext, name }) {
  * Index of promises resolving to File node from remote url
  */
 const processingCache = {}
-/**
- * pushTask
- * --
- * pushes a task in to the Queue and the processing cache
- *
- * Promisfy a task in queue
- * @param {CreateRemoteFileNodePayload} task
- * @return {Promise<Object>}
- */
-const pushTask = task =>
-  new Promise((resolve, reject) => {
-    queue.push(task, (err, node) => {
-      if (!err) {
-        resolve(node)
-      } else {
-        reject(`failed to process ${task.url}\n${err}`)
-      }
-    })
-  })
 
 /***************
  * Entry Point *
@@ -245,11 +190,13 @@ module.exports = function createRemoteFileNode({
 
   if (!url || isWebUri(url) === undefined) {
     return Promise.reject(
-      `url passed to createRemoteFileNode is either missing or not a proper web uri: ${url}`
+      new Error(
+        `url passed to createRemoteFileNode is either missing or not a proper web uri: ${url}`
+      )
     )
   }
 
-  const fileDownloadPromise = pushTask({
+  const fileDownloadPromise = processRemoteNode({
     url,
     cache,
     createNode,