From 00870bbc96e0aaa6b68706ad434129ab0c031154 Mon Sep 17 00:00:00 2001 From: Vladimir Razuvaev Date: Thu, 19 Nov 2020 19:07:19 +0700 Subject: [PATCH] fix(gatsby-source-filesystem): Use new FileType methods to read file extension (#28156) (#28178) (cherry picked from commit 616e9e3e264e58a750f94aaa428cc7803891ae44) Co-authored-by: Peter van der Zee <209817+pvdz@users.noreply.github.com> --- .../gatsby-source-filesystem/package.json | 3 +- .../src/create-file-node-from-buffer.js | 2 +- .../src/create-remote-file-node.js | 4 +- yarn.lock | 91 +++++++++++++++---- 4 files changed, 74 insertions(+), 26 deletions(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index e955b68ccb11c..88389021d06ff 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -10,7 +10,7 @@ "@babel/runtime": "^7.12.5", "better-queue": "^3.8.10", "chokidar": "^3.4.2", - "file-type": "^12.4.2", + "file-type": "^16.0.0", "fs-extra": "^8.1.0", "gatsby-core-utils": "^1.5.0-next.0", "got": "^9.6.0", @@ -18,7 +18,6 @@ "mime": "^2.4.6", "pretty-bytes": "^5.4.1", "progress": "^2.0.3", - "read-chunk": "^3.2.0", "valid-url": "^1.0.9", "xstate": "^4.13.0" }, diff --git a/packages/gatsby-source-filesystem/src/create-file-node-from-buffer.js b/packages/gatsby-source-filesystem/src/create-file-node-from-buffer.js index 7f624e30bc64e..b06e460893997 100644 --- a/packages/gatsby-source-filesystem/src/create-file-node-from-buffer.js +++ b/packages/gatsby-source-filesystem/src/create-file-node-from-buffer.js @@ -70,7 +70,7 @@ async function processBufferNode({ // If the user did not provide an extension and we couldn't get // one from remote file, try and guess one if (typeof ext === `undefined`) { - const filetype = fileType(buffer) + const filetype = await fileType.fromBuffer(buffer) ext = filetype ? `.${filetype.ext}` : `.bin` } diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index f3455ed95a634..7629b59e5557e 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -4,7 +4,6 @@ const { createContentDigest } = require(`gatsby-core-utils`) const path = require(`path`) const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) -const readChunk = require(`read-chunk`) const fileType = require(`file-type`) const { createProgress } = require(`./utils`) @@ -290,8 +289,7 @@ async function fetchRemoteNode({ if (ext === ``) { if (response.statusCode === 200) { // if this is fresh response - try to guess extension and cache result for future - const buffer = readChunk.sync(tmpFilename, 0, fileType.minimumBytes) - const filetype = fileType(buffer) + const filetype = await fileType.fromFile(tmpFilename) if (filetype) { ext = `.${filetype.ext}` await cache.set(cacheIdForExtensions(url), ext) diff --git a/yarn.lock b/yarn.lock index d19a925bcc66d..0ff74acdbf262 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3674,6 +3674,11 @@ "@theme-ui/core" "^0.4.0-rc.7" "@theme-ui/mdx" "^0.4.0-rc.7" +"@tokenizer/token@^0.1.0", "@tokenizer/token@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.1.1.tgz#f0d92c12f87079ddfd1b29f614758b9696bc29e3" + integrity sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w== + "@turist/fetch@^7.1.7": version "7.1.7" resolved "https://registry.yarnpkg.com/@turist/fetch/-/fetch-7.1.7.tgz#a2b1f7ec0265e6fe0946c51eef34bad9b9efc865" @@ -3822,6 +3827,11 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-0.0.30.tgz#dc1e40f7af3b9c815013a7860e6252f6352a84df" integrity sha512-orGL5LXERPYsLov6CWs3Fh6203+dXzJkR7OnddIr2514Hsecwc8xRpzCapshBbKFImCsvS/mk6+FWiN5LyZJAQ== +"@types/debug@^4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" + integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d" @@ -4160,6 +4170,14 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/readable-stream@^2.3.9": + version "2.3.9" + resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-2.3.9.tgz#40a8349e6ace3afd2dd1b6d8e9b02945de4566a9" + integrity sha512-sqsgQqFT7HmQz/V5jH1O0fvQQnXAJO46Gg9LRO/JPfjmVmGUlcx831TZZO3Y3HtWhIkzf3kTsNT0Z0kzIhIvZw== + dependencies: + "@types/node" "*" + safe-buffer "*" + "@types/reflexbox@^4.0.0": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/reflexbox/-/reflexbox-4.0.1.tgz#dfe91aace3c931766507cfd1cce65498a4d052a0" @@ -10752,11 +10770,21 @@ file-type@5.2.0, file-type@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-5.2.0.tgz#2ddbea7c73ffe36368dfae49dc338c058c2b8ad6" -file-type@^12.0.0, file-type@^12.4.2: +file-type@^12.0.0: version "12.4.2" resolved "https://registry.yarnpkg.com/file-type/-/file-type-12.4.2.tgz#a344ea5664a1d01447ee7fb1b635f72feb6169d9" integrity sha512-UssQP5ZgIOKelfsaB5CuGAL+Y+q7EmONuiwF3N5HAH0t27rvrttgi6Ra9k/+DVaY9UF6+ybxu5pOXLUdA8N7Vg== +file-type@^16.0.0: + version "16.0.1" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-16.0.1.tgz#d12d19c716e49d79e0a27bcf502961498b0b9928" + integrity sha512-rwXqMZiizJd0uXZE52KN2DtPBAV99qz9cUTHHt8pSyaQzgVYrHJGR0qt2p4N/yzHEL/tGrlB/TgawQb4Fnxxyw== + dependencies: + readable-web-to-node-stream "^3.0.0" + strtok3 "^6.0.3" + token-types "^2.0.0" + typedarray-to-buffer "^3.1.5" + file-type@^3.8.0: version "3.9.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9" @@ -12989,6 +13017,11 @@ idb-keyval@^3.2.0: resolved "https://registry.yarnpkg.com/idb-keyval/-/idb-keyval-3.2.0.tgz#cbbf354deb5684b6cdc84376294fc05932845bd6" integrity sha512-slx8Q6oywCCSfKgPgL0sEsXtPVnSbTLWpyiDcu6msHOyKOLari1TD1qocXVCft80umnkk3/Qqh3lwoFt8T/BPQ== +ieee754@^1.1.13: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + ieee754@^1.1.4: version "1.1.12" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.12.tgz#50bf24e5b9c8bb98af4964c941cdb0918da7b60b" @@ -18287,7 +18320,7 @@ p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" -p-try@^2.0.0, p-try@^2.1.0: +p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== @@ -18726,6 +18759,11 @@ pdf2json@^1.2.0: optimist "^0.6.1" xmldom "^0.3.0" +peek-readable@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-3.1.0.tgz#250b08b7de09db8573d7fd8ea475215bbff14348" + integrity sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA== + pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" @@ -20160,14 +20198,6 @@ react@^16.12.0, react@^16.8.0: object-assign "^4.1.1" prop-types "^15.6.2" -read-chunk@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/read-chunk/-/read-chunk-3.2.0.tgz#2984afe78ca9bfbbdb74b19387bf9e86289c16ca" - integrity sha512-CEjy9LCzhmD7nUpJ1oVOE6s/hBkejlcJEgLQHVnQznOSilOPb+kpKktlLfFDK3/WP43+F80xkUTM2VOkYoSYvQ== - dependencies: - pify "^4.0.1" - with-open-file "^0.1.6" - read-cmd-shim@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/read-cmd-shim/-/read-cmd-shim-1.0.1.tgz#2d5d157786a37c055d22077c32c53f8329e91c7b" @@ -20308,7 +20338,7 @@ readable-stream@1.0, readable-stream@~1.0.31: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@2 || 3", readable-stream@^3.0.1, readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0: +"readable-stream@2 || 3", readable-stream@^3.0.1, readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -20340,6 +20370,14 @@ readable-stream@~2.1.0: string_decoder "~0.10.x" util-deprecate "~1.0.1" +readable-web-to-node-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.0.tgz#4ca5408e70471069119d691934141a52de413955" + integrity sha512-HNmLb3n0SteGAs8HQlErYPGeO+y7cvL/mVUKtXeUkl0iCZ/2GIgKGrCFHyS7UXFnO8uc9U+0y3pYIzAPsjFfvA== + dependencies: + "@types/readable-stream" "^2.3.9" + readable-stream "^3.6.0" + readdir-scoped-modules@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.0.2.tgz#9fafa37d286be5d92cbaebdee030dc9b5f406747" @@ -22167,6 +22205,11 @@ sade@^1.7.3: dependencies: mri "^1.1.0" +safe-buffer@*: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -23529,6 +23572,15 @@ strong-log-transformer@^2.0.0: minimist "^1.2.0" through "^2.3.4" +strtok3@^6.0.3: + version "6.0.4" + resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.0.4.tgz#ede0d20fde5aa9fda56417c3558eaafccc724694" + integrity sha512-rqWMKwsbN9APU47bQTMEYTPcwdpKDtmf1jVhHzNW2cL1WqAxaM9iBb9t5P2fj+RV2YsErUWgQzHD5JwV0uCTEQ== + dependencies: + "@tokenizer/token" "^0.1.1" + "@types/debug" "^4.1.5" + peek-readable "^3.1.0" + style-inject@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/style-inject/-/style-inject-0.3.0.tgz#d21c477affec91811cc82355832a700d22bf8dd3" @@ -24316,6 +24368,14 @@ toidentifier@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" +token-types@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/token-types/-/token-types-2.0.0.tgz#b23618af744818299c6fbf125e0fdad98bab7e85" + integrity sha512-WWvu8sGK8/ZmGusekZJJ5NM6rRVTTDO7/bahz4NGiSDb/XsmdYBn6a1N/bymUHuWYTWeuLUg98wUzvE4jPdCZw== + dependencies: + "@tokenizer/token" "^0.1.0" + ieee754 "^1.1.13" + toml@^2.3.2, toml@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/toml/-/toml-2.3.6.tgz#25b0866483a9722474895559088b436fd11f861b" @@ -25952,15 +26012,6 @@ windows-release@^3.1.0: dependencies: execa "^1.0.0" -with-open-file@^0.1.6: - version "0.1.6" - resolved "https://registry.yarnpkg.com/with-open-file/-/with-open-file-0.1.6.tgz#0bc178ecab75f6baac8ae11c85e07445d690ea50" - integrity sha512-SQS05JekbtwQSgCYlBsZn/+m2gpn4zWsqpCYIrCHva0+ojXcnmUEPsBN6Ipoz3vmY/81k5PvYEWSxER2g4BTqA== - dependencies: - p-finally "^1.0.0" - p-try "^2.1.0" - pify "^4.0.1" - wmf@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/wmf/-/wmf-1.0.1.tgz#f8690f185651bf88d39f0a21ae3e51bb1ec9fae9"