From 42264599e3b88a1a723be7429db987994b1a7111 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 26 Feb 2021 18:18:16 -0800 Subject: [PATCH 1/5] copy missing progress fn (this is temporary) --- packages/gatsby-source-wordpress/package.json | 3 +- .../create-progress.js | 91 +++++++++++++++++++ .../create-remote-file-node/index.js | 2 +- yarn.lock | 34 ++++++- 4 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js diff --git a/packages/gatsby-source-wordpress/package.json b/packages/gatsby-source-wordpress/package.json index 57e5a75f53cdb..a6a083a6d3e27 100644 --- a/packages/gatsby-source-wordpress/package.json +++ b/packages/gatsby-source-wordpress/package.json @@ -11,6 +11,7 @@ "@rematch/core": "^1.3.0", "@rematch/immer": "^1.2.0", "async-retry": "^1.3.1", + "progress": "2.0.3", "atob": "^2.1.2", "axios": "^0.21.1", "axios-rate-limit": "^1.3.0", @@ -31,7 +32,7 @@ "gatsby-core-utils": "^2.1.0-next.1", "gatsby-image": "^3.1.0-next.1", "gatsby-plugin-catch-links": "^3.1.0-next.1", - "gatsby-source-filesystem": "^3.1.0-next.1", + "gatsby-source-filesystem": "2.11.1", "glob": "^7.1.6", "got": "^11.7.0", "graphql-query-compress": "^1.2.2", diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js new file mode 100644 index 0000000000000..b6a5eef31c2aa --- /dev/null +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js @@ -0,0 +1,91 @@ +exports.__esModule = true +exports.getRemoteFileExtension = getRemoteFileExtension +exports.getRemoteFileName = getRemoteFileName +exports.createProgress = createProgress +exports.createFilePath = createFilePath + +const path = require(`path`) + +const Url = require(`url`) + +const ProgressBar = require(`progress`) +/** + * getParsedPath + * -- + * Parses remote url to a path object + * + * + * @param {String} url + * @return {Object} path + */ + +function getParsedPath(url) { + return path.parse(Url.parse(url).pathname) +} +/** + * getRemoteFileExtension + * -- + * Parses remote url to retrieve remote file extension + * + * + * @param {String} url + * @return {String} extension + */ + +function getRemoteFileExtension(url) { + return getParsedPath(url).ext +} +/** + * getRemoteFileName + * -- + * Parses remote url to retrieve remote file name + * + * + * @param {String} url + * @return {String} filename + */ + +function getRemoteFileName(url) { + return getParsedPath(url).name +} // TODO remove in V3 + +function createProgress(message, reporter) { + if (reporter && reporter.createProgress) { + return reporter.createProgress(message) + } + + const bar = new ProgressBar( + ` [:bar] :current/:total :elapsed s :percent ${message}`, + { + total: 0, + width: 30, + clear: true, + } + ) + return { + start() {}, + + tick() { + bar.tick() + }, + + done() {}, + + set total(value) { + bar.total = value + }, + } +} +/** + * createFilePath + * -- + * + * @param {String} directory + * @param {String} filename + * @param {String} ext + * @return {String} + */ + +function createFilePath(directory, filename, ext) { + return path.join(directory, `${filename}${ext}`) +} diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js index 9272667adaba5..4c6759ce59444 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js @@ -8,7 +8,7 @@ const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) const readChunk = require(`read-chunk`) const fileType = require(`file-type`) -const { createProgress } = require(`gatsby-source-filesystem/utils`) +const { createProgress } = require(`./create-progress`) const { createFileNode } = require(`gatsby-source-filesystem/create-file-node`) const { diff --git a/yarn.lock b/yarn.lock index ab3081c352d39..b502b43c16aa3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13177,6 +13177,19 @@ gather-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/gather-stream/-/gather-stream-1.0.0.tgz#b33994af457a8115700d410f317733cbe7a0904b" +gatsby-core-utils@^1.10.1: + version "1.10.1" + resolved "https://registry.yarnpkg.com/gatsby-core-utils/-/gatsby-core-utils-1.10.1.tgz#97bed40df3fa79800e7ce0c0491680f0aadd6ce7" + integrity sha512-4P3feGCJckg+DRWWl2beFk7N9c63zmCryEGPaU1OHCp+ZT2bO0ihCBuXywDWuuEp6SYP9PZ1fs0YJ/Rt6q6lag== + dependencies: + ci-info "2.0.0" + configstore "^5.0.1" + fs-extra "^8.1.0" + node-object-hash "^2.0.0" + proper-lockfile "^4.1.1" + tmp "^0.2.1" + xdg-basedir "^4.0.0" + gatsby-design-tokens@^2.0.2: version "2.0.13" resolved "https://registry.yarnpkg.com/gatsby-design-tokens/-/gatsby-design-tokens-2.0.13.tgz#33f5fa84a399b821ae224b9921847d7b37c45600" @@ -13227,6 +13240,25 @@ gatsby-plugin-webfonts@^1.1.4: postcss "^8.2.2" postcss-js "^3.0.3" +gatsby-source-filesystem@2.11.1: + version "2.11.1" + resolved "https://registry.yarnpkg.com/gatsby-source-filesystem/-/gatsby-source-filesystem-2.11.1.tgz#c90f49c096c4bdafd8e68462966b3a6e21fd746b" + integrity sha512-Ao526Mmhm8KkF+0Tvf9Le5kKnqX7kgC1wecp82BW2KLQgdtG7UIgmHvG6PkjuFNiJ2ghXPC3vRK3J/vDPyLtkA== + dependencies: + "@babel/runtime" "^7.12.5" + better-queue "^3.8.10" + chokidar "^3.4.3" + file-type "^16.0.0" + fs-extra "^8.1.0" + gatsby-core-utils "^1.10.1" + got "^9.6.0" + md5-file "^5.0.0" + mime "^2.4.6" + pretty-bytes "^5.4.1" + progress "^2.0.3" + valid-url "^1.0.9" + xstate "^4.14.0" + gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -21544,7 +21576,7 @@ process@~0.5.1: version "0.5.2" resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf" -progress@^2.0.0, progress@^2.0.1, progress@^2.0.3: +progress@2.0.3, progress@^2.0.0, progress@^2.0.1, progress@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" From a9b139c2ef1e84818446fbd5193a8d4619c1ad8e Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 26 Feb 2021 18:18:32 -0800 Subject: [PATCH 2/5] prevent resolver from breaking when no existing node exists --- .../transform-fields/transform-object.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-wordpress/src/steps/create-schema-customization/transform-fields/transform-object.js b/packages/gatsby-source-wordpress/src/steps/create-schema-customization/transform-fields/transform-object.js index cc33a60d30d3e..d3940964e6d26 100644 --- a/packages/gatsby-source-wordpress/src/steps/create-schema-customization/transform-fields/transform-object.js +++ b/packages/gatsby-source-wordpress/src/steps/create-schema-customization/transform-fields/transform-object.js @@ -54,7 +54,7 @@ export const buildGatsbyNodeObjectResolver = ({ field, fieldName }) => async ( schema: { typePrefix: prefix }, } = getPluginOptions() - if (!existingNode.__typename.startsWith(prefix)) { + if (existingNode?.__typename && !existingNode.__typename.startsWith(prefix)) { existingNode.__typename = buildTypeName(existingNode.__typename) } From 1a61ee71c5440c3c289ba3e4f756509638aeba51 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 26 Feb 2021 18:19:20 -0800 Subject: [PATCH 3/5] replacing hompage links can break images.. only replace if they don't include wp-content, etc --- .../src/steps/source-nodes/create-nodes/process-node.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/process-node.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/process-node.js index f2cb23da72615..8ac48aa5fc9a1 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/process-node.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/process-node.js @@ -827,7 +827,11 @@ const replaceNodeHtmlLinks = ({ wpUrl, nodeString, node }) => { const normalizedPath = path.replace(/\\/g, ``) // replace normalized match with relative path - const thisMatchRegex = new RegExp(normalizedMatch, `g`) + const thisMatchRegex = new RegExp( + normalizedMatch + `(?!/?wp-content|/?wp-admin|/?wp-includes)`, + `g` + ) + nodeString = nodeString.replace(thisMatchRegex, normalizedPath) } catch (e) { console.error(e) From d484ec4fa268e028254215be8e4f2c50379e585f Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 26 Feb 2021 18:39:04 -0800 Subject: [PATCH 4/5] don't swallow errors - don't use broken reporter.panic --- .../source-nodes/create-nodes/create-remote-media-item-node.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-media-item-node.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-media-item-node.js index 2679398744024..f35af40b69efc 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-media-item-node.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-media-item-node.js @@ -138,6 +138,9 @@ export const errorPanicker = ({ ) ) reporter.panic(error) + } else { + console.error(error) + reporter.panic() } } From 9d45e01cbd83a7104bafbbb345a3cb2e1268fbfa Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 1 Mar 2021 12:00:13 +0100 Subject: [PATCH 5/5] use reporter.createProgress --- packages/gatsby-source-wordpress/package.json | 3 +- .../create-progress.js | 91 ------------------- .../create-remote-file-node/index.js | 3 +- yarn.lock | 34 +------ 4 files changed, 3 insertions(+), 128 deletions(-) delete mode 100644 packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js diff --git a/packages/gatsby-source-wordpress/package.json b/packages/gatsby-source-wordpress/package.json index a6a083a6d3e27..57e5a75f53cdb 100644 --- a/packages/gatsby-source-wordpress/package.json +++ b/packages/gatsby-source-wordpress/package.json @@ -11,7 +11,6 @@ "@rematch/core": "^1.3.0", "@rematch/immer": "^1.2.0", "async-retry": "^1.3.1", - "progress": "2.0.3", "atob": "^2.1.2", "axios": "^0.21.1", "axios-rate-limit": "^1.3.0", @@ -32,7 +31,7 @@ "gatsby-core-utils": "^2.1.0-next.1", "gatsby-image": "^3.1.0-next.1", "gatsby-plugin-catch-links": "^3.1.0-next.1", - "gatsby-source-filesystem": "2.11.1", + "gatsby-source-filesystem": "^3.1.0-next.1", "glob": "^7.1.6", "got": "^11.7.0", "graphql-query-compress": "^1.2.2", diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js deleted file mode 100644 index b6a5eef31c2aa..0000000000000 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/create-progress.js +++ /dev/null @@ -1,91 +0,0 @@ -exports.__esModule = true -exports.getRemoteFileExtension = getRemoteFileExtension -exports.getRemoteFileName = getRemoteFileName -exports.createProgress = createProgress -exports.createFilePath = createFilePath - -const path = require(`path`) - -const Url = require(`url`) - -const ProgressBar = require(`progress`) -/** - * getParsedPath - * -- - * Parses remote url to a path object - * - * - * @param {String} url - * @return {Object} path - */ - -function getParsedPath(url) { - return path.parse(Url.parse(url).pathname) -} -/** - * getRemoteFileExtension - * -- - * Parses remote url to retrieve remote file extension - * - * - * @param {String} url - * @return {String} extension - */ - -function getRemoteFileExtension(url) { - return getParsedPath(url).ext -} -/** - * getRemoteFileName - * -- - * Parses remote url to retrieve remote file name - * - * - * @param {String} url - * @return {String} filename - */ - -function getRemoteFileName(url) { - return getParsedPath(url).name -} // TODO remove in V3 - -function createProgress(message, reporter) { - if (reporter && reporter.createProgress) { - return reporter.createProgress(message) - } - - const bar = new ProgressBar( - ` [:bar] :current/:total :elapsed s :percent ${message}`, - { - total: 0, - width: 30, - clear: true, - } - ) - return { - start() {}, - - tick() { - bar.tick() - }, - - done() {}, - - set total(value) { - bar.total = value - }, - } -} -/** - * createFilePath - * -- - * - * @param {String} directory - * @param {String} filename - * @param {String} ext - * @return {String} - */ - -function createFilePath(directory, filename, ext) { - return path.join(directory, `${filename}${ext}`) -} diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js index 4c6759ce59444..ee0b995c14b02 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/create-remote-file-node/index.js @@ -8,7 +8,6 @@ const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) const readChunk = require(`read-chunk`) const fileType = require(`file-type`) -const { createProgress } = require(`./create-progress`) const { createFileNode } = require(`gatsby-source-filesystem/create-file-node`) const { @@ -431,7 +430,7 @@ module.exports = ({ } if (totalJobs === 0) { - bar = createProgress(`Downloading remote files`, reporter) + bar = reporter.createProgress(`Downloading remote files`) bar.start() } diff --git a/yarn.lock b/yarn.lock index b502b43c16aa3..ab3081c352d39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13177,19 +13177,6 @@ gather-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/gather-stream/-/gather-stream-1.0.0.tgz#b33994af457a8115700d410f317733cbe7a0904b" -gatsby-core-utils@^1.10.1: - version "1.10.1" - resolved "https://registry.yarnpkg.com/gatsby-core-utils/-/gatsby-core-utils-1.10.1.tgz#97bed40df3fa79800e7ce0c0491680f0aadd6ce7" - integrity sha512-4P3feGCJckg+DRWWl2beFk7N9c63zmCryEGPaU1OHCp+ZT2bO0ihCBuXywDWuuEp6SYP9PZ1fs0YJ/Rt6q6lag== - dependencies: - ci-info "2.0.0" - configstore "^5.0.1" - fs-extra "^8.1.0" - node-object-hash "^2.0.0" - proper-lockfile "^4.1.1" - tmp "^0.2.1" - xdg-basedir "^4.0.0" - gatsby-design-tokens@^2.0.2: version "2.0.13" resolved "https://registry.yarnpkg.com/gatsby-design-tokens/-/gatsby-design-tokens-2.0.13.tgz#33f5fa84a399b821ae224b9921847d7b37c45600" @@ -13240,25 +13227,6 @@ gatsby-plugin-webfonts@^1.1.4: postcss "^8.2.2" postcss-js "^3.0.3" -gatsby-source-filesystem@2.11.1: - version "2.11.1" - resolved "https://registry.yarnpkg.com/gatsby-source-filesystem/-/gatsby-source-filesystem-2.11.1.tgz#c90f49c096c4bdafd8e68462966b3a6e21fd746b" - integrity sha512-Ao526Mmhm8KkF+0Tvf9Le5kKnqX7kgC1wecp82BW2KLQgdtG7UIgmHvG6PkjuFNiJ2ghXPC3vRK3J/vDPyLtkA== - dependencies: - "@babel/runtime" "^7.12.5" - better-queue "^3.8.10" - chokidar "^3.4.3" - file-type "^16.0.0" - fs-extra "^8.1.0" - gatsby-core-utils "^1.10.1" - got "^9.6.0" - md5-file "^5.0.0" - mime "^2.4.6" - pretty-bytes "^5.4.1" - progress "^2.0.3" - valid-url "^1.0.9" - xstate "^4.14.0" - gauge@~2.7.3: version "2.7.4" resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" @@ -21576,7 +21544,7 @@ process@~0.5.1: version "0.5.2" resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf" -progress@2.0.3, progress@^2.0.0, progress@^2.0.1, progress@^2.0.3: +progress@^2.0.0, progress@^2.0.1, progress@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"