diff --git a/integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js b/integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js new file mode 100644 index 0000000000000..7411d693736d9 --- /dev/null +++ b/integration-tests/gatsby-pipeline/__tests__/fetch-remote-file/index.js @@ -0,0 +1,63 @@ +/** + * We want to make sure that fetch-remote-file is working with multi workers. + */ + +const execa = require(`execa`) +const path = require(`path`) +const glob = require(`glob`) +const fs = require(`fs-extra`) +const md5File = require(`md5-file`) +const basePath = path.resolve(__dirname, `../../`) + +const cleanDirs = () => + Promise.all([ + fs.emptyDir(`${basePath}/public`), + fs.emptyDir(`${basePath}/.cache`), + ]) + +describe(`fetch-remote-file`, () => { + beforeAll(async () => { + await cleanDirs() + await execa(`yarn`, [`build`], { + cwd: basePath, + // we want to force 1 query per worker + env: { NODE_ENV: `production`, GATSBY_PARALLEL_QUERY_CHUNK_SIZE: `1` }, + }) + }, 60 * 1000) + + it("should have the correct md5", async () => { + expect( + await md5File( + path.join( + __dirname, + "../..", + "public/images/50c58a791de3c2303e62084d731799eb/photoA.jpg" + ) + ) + ).toEqual("a9e57a66a10b2d26a1999a4685d7c9ef") + expect( + await md5File( + path.join( + __dirname, + "../..", + "public/images/4910e745c3c453b8795d6ba65c79d99b/photoB.jpg" + ) + ) + ).toEqual("c305dc5c5db45cc773231a507af5116d") + expect( + await md5File( + path.join( + __dirname, + "../..", + "public/images/fb673e75e9534b3cc2d2e24085386d48/photoC.jpg" + ) + ) + ).toEqual("4ba953ba27236727d7abe7d5b8916432") + }) + + it("should have conflict between workers", async () => { + const files = await fs.readdir(path.join(__dirname, "../../.cache/workers")) + + expect(files.length).toBe(4) + }) +}) diff --git a/integration-tests/gatsby-pipeline/gatsby-node.js b/integration-tests/gatsby-pipeline/gatsby-node.js new file mode 100644 index 0000000000000..07cf62385916c --- /dev/null +++ b/integration-tests/gatsby-pipeline/gatsby-node.js @@ -0,0 +1,69 @@ +const { fetchRemoteFile } = require("gatsby-core-utils/fetch-remote-file") +const { slash } = require("gatsby-core-utils") +const path = require("path") +const fs = require("fs-extra") + +/** @type{import('gatsby').createSchemaCustomization} */ +exports.createSchemaCustomization = ({ actions, schema, cache, reporter }) => { + actions.createTypes( + schema.buildObjectType({ + name: "MyRemoteFile", + fields: { + url: "String!", + publicUrl: { + type: "String!", + async resolve(source) { + const filePath = await fetchRemoteFile({ + name: path.basename(source.name, path.extname(source.name)), + ext: path.extname(source.name), + url: source.url, + directory: "./public/images", + }) + + const dir = path.join(global.__GATSBY.root, ".cache", "workers") + await fs.ensureDir(dir) + await fs.createFile( + `${path.join(dir, `worker-${process.env.GATSBY_WORKER_ID}`)}` + ) + + const workers = (await cache.get("workers")) ?? [] + workers.push(process.env.GATSBY_WORKER_ID) + + return `${slash(filePath.replace(/^public/, ""))}` + }, + }, + }, + interfaces: ["Node"], + }) + ) +} + +/** @type {imporg('gatsby').sourceNodes} */ +exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => { + const items = [ + { + name: "photoA.jpg", + url: "https://images.unsplash.com/photo-1517849845537-4d257902454a?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80", + }, + { + name: "photoB.jpg", + url: "https://images.unsplash.com/photo-1552053831-71594a27632d?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80", + }, + { + name: "photoC.jpg", + url: "https://images.unsplash.com/photo-1561037404-61cd46aa615b?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=300&q=80", + }, + ] + + items.forEach((item, index) => { + actions.createNode({ + id: createNodeId(`remote-file-${index}`), + name: item.name, + url: item.url, + internal: { + type: "MyRemoteFile", + contentDigest: createContentDigest(item.url), + }, + }) + }) +} diff --git a/integration-tests/gatsby-pipeline/package.json b/integration-tests/gatsby-pipeline/package.json index bab9e890a76b1..4d723a8bec92a 100644 --- a/integration-tests/gatsby-pipeline/package.json +++ b/integration-tests/gatsby-pipeline/package.json @@ -4,13 +4,13 @@ "version": "1.0.0", "author": "Kyle Mathews ", "dependencies": { - "gatsby": "latest", + "gatsby": "4.8.0-next.1-dev-1645011114818", "gatsby-image": "latest", - "gatsby-plugin-gatsby-cloud": "latest", - "gatsby-plugin-react-helmet": "latest", - "gatsby-plugin-sharp": "latest", - "gatsby-source-filesystem": "latest", - "gatsby-transformer-sharp": "latest", + "gatsby-plugin-gatsby-cloud": "4.8.0-next.1-dev-1645011114818", + "gatsby-plugin-react-helmet": "5.8.0-next.0-dev-1645011114818", + "gatsby-plugin-sharp": "4.8.0-next.0-dev-1645011114818", + "gatsby-source-filesystem": "4.8.0-next.0-dev-1645011114818", + "gatsby-transformer-sharp": "4.8.0-next.0-dev-1645011114818", "prop-types": "^15.7.2", "react": "^17.0.2", "react-dom": "^17.0.2", @@ -26,11 +26,12 @@ "test": "jest" }, "devDependencies": { + "cross-env": "^7.0.3", "execa": "^4.0.3", "fs-extra": "^9.0.1", + "jest": "^27.2.1", "md5-file": "^5.0.0", "node-fetch": "^2.6.0", - "jest": "^27.2.1", "tree-kill": "^1.2.2" }, "repository": { @@ -40,4 +41,4 @@ "engines": { "node": ">=12.13.0" } -} +} \ No newline at end of file diff --git a/integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js new file mode 100644 index 0000000000000..7163b394c86d0 --- /dev/null +++ b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-a.js @@ -0,0 +1,32 @@ +import React from "react" +import { graphql, Link } from "gatsby" + +import Layout from "../components/layout" +import SEO from "../components/seo" + +const FetchRemoteA = ({ data }) => { + console.log({ data }) + return ( + + + + 
+      Go back to the homepage
+    
+  )
+}
+
+export default FetchRemoteA
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`
diff --git a/integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js
new file mode 100644
index 0000000000000..e7e86751904a4
--- /dev/null
+++ b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-b.js
@@ -0,0 +1,32 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    
+      
+
+      
+
+      Go back to the homepage
+    
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`
diff --git a/integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js
new file mode 100644
index 0000000000000..6e3535c399090
--- /dev/null
+++ b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-c.js
@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    
+      
+
+      
+      Go back to the homepage
+    
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`
diff --git a/integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js
new file mode 100644
index 0000000000000..88312818acc3c
--- /dev/null
+++ b/integration-tests/gatsby-pipeline/src/pages/fetch-remote-d.js
@@ -0,0 +1,31 @@
+import React from "react"
+import { graphql, Link } from "gatsby"
+
+import Layout from "../components/layout"
+import SEO from "../components/seo"
+
+const FetchRemoteB = ({ data }) => {
+  return (
+    
+      
+
+      
+      Go back to the homepage
+    
+  )
+}
+
+export default FetchRemoteB
+
+export const pageQuery = graphql`
+  {
+    allMyRemoteFile {
+      nodes {
+        url
+        publicUrl
+      }
+    }
+  }
+`
diff --git a/packages/gatsby-core-utils/src/fetch-remote-file.ts b/packages/gatsby-core-utils/src/fetch-remote-file.ts
index ddca9e5aba6c3..29cbdce4aa2c9 100644
--- a/packages/gatsby-core-utils/src/fetch-remote-file.ts
+++ b/packages/gatsby-core-utils/src/fetch-remote-file.ts
@@ -8,6 +8,7 @@ import {
   getRemoteFileExtension,
   createFilePath,
 } from "./filename-utils"
+import { slash } from "./path"
 import { requestRemoteNode } from "./remote-file-utils/fetch-file"
 import { getStorage, getDatabaseDir } from "./utils/get-storage"
 import { createMutex } from "./mutex"
@@ -182,7 +183,9 @@ async function fetchFile({
       httpOptions
     )
 
-    const filename = createFilePath(path.join(fileDirectory, digest), name, ext)
+    const filename = slash(
+      createFilePath(path.join(fileDirectory, digest), name, ext)
+    )
     if (response.statusCode === 200) {
       // Save the response headers for future requests.
       // If the user did not provide an extension and we couldn't get one from remote file, try and guess one
@@ -200,7 +203,7 @@ async function fetchFile({
         cacheKey,
         extension: ext,
         headers: response.headers.etag ? { etag: response.headers.etag } : {},
-        directory: fileDirectory,
+        directory: slash(fileDirectory),
         path: filename.replace(fileDirectory, ``),
       })
     } else if (response.statusCode === 304) {