diff --git a/docs/docs/schema-customization.md b/docs/docs/schema-customization.md
index 95a2832ba7703..2261a2aecee8b 100644
--- a/docs/docs/schema-customization.md
+++ b/docs/docs/schema-customization.md
@@ -801,6 +801,7 @@ exports.createResolvers = ({
createNodeId,
createResolvers,
store,
+ reporter,
}) => {
const { createNode } = actions
createResolvers({
@@ -814,6 +815,7 @@ exports.createResolvers = ({
cache,
createNode,
createNodeId,
+ reporter,
})
},
},
diff --git a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js
index 5b5a86be28eed..7aedba97bc89c 100644
--- a/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js
+++ b/examples/using-gatsby-image/plugins/gatsby-source-remote-images/gatsby-node.js
@@ -1,7 +1,14 @@
const { createRemoteFileNode } = require(`gatsby-source-filesystem`)
exports.onCreateNode = async (
- { actions: { createNode }, node, createContentDigest, store, cache },
+ {
+ actions: { createNode },
+ node,
+ createContentDigest,
+ store,
+ cache,
+ reporter,
+ },
{ filter, nodeName = `localFile` }
) => {
if (filter(node)) {
@@ -11,6 +18,7 @@ exports.onCreateNode = async (
cache,
createNode,
createNodeId: createContentDigest,
+ reporter,
})
if (fileNode) {
diff --git a/examples/using-gatsby-source-graphql/gatsby-node.js b/examples/using-gatsby-source-graphql/gatsby-node.js
index e7009b6515996..51cb7449db808 100644
--- a/examples/using-gatsby-source-graphql/gatsby-node.js
+++ b/examples/using-gatsby-source-graphql/gatsby-node.js
@@ -35,6 +35,7 @@ exports.createResolvers = ({
createNodeId,
createResolvers,
store,
+ reporter,
}) => {
const { createNode } = actions
createResolvers({
@@ -64,6 +65,7 @@ exports.createResolvers = ({
cache,
createNode,
createNodeId,
+ reporter,
})
},
},
diff --git a/packages/gatsby-cli/package.json b/packages/gatsby-cli/package.json
index e4f3a852bf4e5..f6541bc2b794b 100644
--- a/packages/gatsby-cli/package.json
+++ b/packages/gatsby-cli/package.json
@@ -33,6 +33,7 @@
"object.entries": "^1.1.0",
"opentracing": "^0.14.3",
"pretty-error": "^2.1.1",
+ "progress": "^2.0.3",
"prompts": "^2.1.0",
"react": "^16.8.4",
"resolve-cwd": "^2.0.0",
diff --git a/packages/gatsby-cli/src/reporter/index.js b/packages/gatsby-cli/src/reporter/index.js
index d0d4c4e71bd6b..37d2581092869 100644
--- a/packages/gatsby-cli/src/reporter/index.js
+++ b/packages/gatsby-cli/src/reporter/index.js
@@ -93,13 +93,88 @@ const reporter: Reporter = {
const spanArgs = parentSpan ? { childOf: parentSpan } : {}
const span = tracer.startSpan(name, spanArgs)
- const activity = reporterInstance.createActivity(name)
+ const activity = reporterInstance.createActivity({
+ type: `spinner`,
+ id: name,
+ status: ``,
+ })
return {
- ...activity,
+ start() {
+ activity.update({
+ startTime: process.hrtime(),
+ })
+ },
+ setStatus(status) {
+ activity.update({
+ status: status,
+ })
+ },
end() {
span.finish()
- activity.end()
+ activity.done()
+ },
+ span,
+ }
+ },
+
+ /**
+ * Create a progress bar for an activity
+ * @param {string} name - Name of activity.
+ * @param {number} total - Total items to be processed.
+ * @param {number} start - Start count to show.
+ * @param {ActivityArgs} activityArgs - optional object with tracer parentSpan
+ * @returns {ActivityTracker} The activity tracker.
+ */
+ createProgress(
+ name: string,
+ total,
+ start = 0,
+ activityArgs: ActivityArgs = {}
+ ): ActivityTracker {
+ const { parentSpan } = activityArgs
+ const spanArgs = parentSpan ? { childOf: parentSpan } : {}
+ const span = tracer.startSpan(name, spanArgs)
+
+ let hasStarted = false
+ let current = start
+ const activity = reporterInstance.createActivity({
+ type: `progress`,
+ id: name,
+ current,
+ total,
+ })
+
+ return {
+ start() {
+ if (hasStarted) {
+ return
+ }
+
+ hasStarted = true
+ activity.update({
+ startTime: process.hrtime(),
+ })
+ },
+ setStatus(status) {
+ activity.update({
+ status: status,
+ })
+ },
+ tick() {
+ activity.update({
+ current: ++current,
+ })
+ },
+ done() {
+ span.finish()
+ activity.done()
+ },
+ set total(value) {
+ total = value
+ activity.update({
+ total: value,
+ })
},
span,
}
diff --git a/packages/gatsby-cli/src/reporter/reporters/ink/components/progress-bar.js b/packages/gatsby-cli/src/reporter/reporters/ink/components/progress-bar.js
new file mode 100644
index 0000000000000..6edc4253cbf1c
--- /dev/null
+++ b/packages/gatsby-cli/src/reporter/reporters/ink/components/progress-bar.js
@@ -0,0 +1,43 @@
+import React from "react"
+import { Box } from "ink"
+import calcElapsedTime from "../../../../util/calc-elapsed-time"
+
+const maxWidth = 30
+const minWidth = 10
+
+const getLength = prop => String(prop).length
+
+export default function ProgressBar({ message, current, total, startTime }) {
+ const percentage = total ? Math.round((current / total) * 100) : 0
+ const terminalWidth = process.stdout.columns || 80
+ const availableWidth =
+ terminalWidth -
+ getLength(message) -
+ getLength(current) -
+ getLength(total) -
+ getLength(percentage) -
+ 11 // margins + extra characters
+
+ const progressBarWidth = Math.max(
+ minWidth,
+ Math.min(maxWidth, availableWidth)
+ )
+
+ return (
+
+
+ [
+
+ {`=`.repeat(((progressBarWidth - 2) * percentage) / 100)}
+
+ ]
+
+ {calcElapsedTime(startTime)} s
+
+ {current}/{total}
+
+ {`` + percentage}%
+ {message}
+
+ )
+}
diff --git a/packages/gatsby-cli/src/reporter/reporters/ink/components/activity.js b/packages/gatsby-cli/src/reporter/reporters/ink/components/spinner.js
similarity index 61%
rename from packages/gatsby-cli/src/reporter/reporters/ink/components/activity.js
rename to packages/gatsby-cli/src/reporter/reporters/ink/components/spinner.js
index 1dcb28fae507b..10f8a8033e57f 100644
--- a/packages/gatsby-cli/src/reporter/reporters/ink/components/activity.js
+++ b/packages/gatsby-cli/src/reporter/reporters/ink/components/spinner.js
@@ -1,14 +1,7 @@
import React from "react"
-import convertHrtime from "convert-hrtime"
import { Box } from "ink"
import Spinner from "ink-spinner"
-export const calcElapsedTime = startTime => {
- const elapsed = process.hrtime(startTime)
-
- return convertHrtime(elapsed)[`seconds`].toFixed(3)
-}
-
export default function Activity({ name, status }) {
let statusText = name
if (status) {
diff --git a/packages/gatsby-cli/src/reporter/reporters/ink/reporter.js b/packages/gatsby-cli/src/reporter/reporters/ink/reporter.js
index f5e220638e742..0c26af36de5dc 100644
--- a/packages/gatsby-cli/src/reporter/reporters/ink/reporter.js
+++ b/packages/gatsby-cli/src/reporter/reporters/ink/reporter.js
@@ -1,19 +1,29 @@
import React from "react"
import { Static, Box } from "ink"
-import { isCI } from "ci-info"
import chalk from "chalk"
-import Activity, { calcElapsedTime } from "./components/activity"
+import Spinner from "./components/spinner"
+import ProgressBar from "./components/progress-bar"
import { Message } from "./components/messages"
+import isTTY from "../../../util/is-tty"
+import calcElapsedTime from "../../../util/calc-elapsed-time"
+
+const showProgress = isTTY
+
+const successTextGenerator = {
+ spinner: activity => {
+ let successText = `${activity.id} - ${calcElapsedTime(
+ activity.startTime
+ )} s`
+ if (activity.status) {
+ successText += ` — ${activity.status}`
+ }
-const showProgress = process.stdout.isTTY && !isCI
-
-const generateActivityFinishedText = (name, activity) => {
- let successText = `${name} - ${calcElapsedTime(activity.startTime)} s`
- if (activity.status) {
- successText += ` — ${activity.status}`
- }
-
- return successText
+ return successText
+ },
+ progress: activity =>
+ `${activity.id} — ${activity.current}/${activity.total} - ${calcElapsedTime(
+ activity.startTime
+ )} s`,
}
export default class GatsbyReporter extends React.Component {
@@ -26,44 +36,40 @@ export default class GatsbyReporter extends React.Component {
format = chalk
- createActivity = name => {
+ createActivity = ({ id, ...options }) => {
+ this.setState(state => {
+ return {
+ activities: {
+ ...state.activities,
+ [id]: {
+ id,
+ ...options,
+ },
+ },
+ }
+ })
+
return {
- start: () => {
- this.setState(state => {
- return {
- activities: {
- ...state.activities,
- [name]: {
- status: ``,
- startTime: process.hrtime(),
- },
- },
- }
- })
- },
- setStatus: status => {
+ update: newState => {
this.setState(state => {
- const activity = state.activities[name]
-
return {
activities: {
...state.activities,
- [name]: {
- ...activity,
- status: status,
+ [id]: {
+ ...state.activities[id],
+ ...newState,
},
},
}
})
},
- end: () => {
- const activity = this.state.activities[name]
-
- this.success(generateActivityFinishedText(name, activity))
+ done: () => {
+ const activity = this.state.activities[id]
+ this.success(successTextGenerator[activity.type]({ id, ...activity }))
this.setState(state => {
const activities = { ...state.activities }
- delete activities[name]
+ delete activities[id]
return {
activities,
@@ -116,27 +122,48 @@ export default class GatsbyReporter extends React.Component {
}
render() {
+ const { activities, messages, disableColors } = this.state
+
+ const spinners = []
+ const progressBars = []
+ if (showProgress) {
+ Object.keys(activities).forEach(activityName => {
+ const activity = activities[activityName]
+ if (activity.type === `spinner`) {
+ spinners.push(activity)
+ }
+ if (activity.type === `progress` && activity.startTime) {
+ progressBars.push(activity)
+ }
+ })
+ }
+
return (
- {this.state.messages.map((msg, index) => (
+ {messages.map((msg, index) => (
-
+
{msg.text}
))}
- {showProgress &&
- Object.keys(this.state.activities).map(activityName => (
-
- ))}
+ {spinners.map(activity => (
+
+ ))}
+
+ {progressBars.map(activity => (
+
+ ))}
)
diff --git a/packages/gatsby-cli/src/reporter/reporters/yurnalist/index.js b/packages/gatsby-cli/src/reporter/reporters/yurnalist/index.js
index 45e3954d5a4fb..537e25eb018a5 100644
--- a/packages/gatsby-cli/src/reporter/reporters/yurnalist/index.js
+++ b/packages/gatsby-cli/src/reporter/reporters/yurnalist/index.js
@@ -1,7 +1,8 @@
// @flow
const { createReporter } = require(`yurnalist`)
-const convertHrtime = require(`convert-hrtime`)
+const ProgressBar = require(`progress`)
+const calcElapsedTime = require(`../../../util/calc-elapsed-time`)
const VERBOSE = process.env.gatsby_log_level === `verbose`
const reporter = createReporter({ emoji: true, verbose: VERBOSE })
@@ -29,36 +30,69 @@ module.exports = {
info: reporter.info.bind(reporter),
warn: reporter.warn.bind(reporter),
log: reporter.log.bind(reporter),
- /**
- * Time an activity.
- * @param {string} name - Name of activity.
- * @returns {string} The elapsed time of activity.
- */
- createActivity(name) {
- const spinner = reporter.activity()
- const start = process.hrtime()
- let status
- const elapsedTime = () => {
- var elapsed = process.hrtime(start)
- return `${convertHrtime(elapsed)[`seconds`].toFixed(3)} s`
+ createActivity: activity => {
+ let start
+
+ if (activity.type === `spinner`) {
+ const spinner = reporter.activity()
+ let status
+
+ return {
+ update: newState => {
+ if (newState.startTime) {
+ start = newState.startTime
+ spinner.tick(activity.id)
+ }
+ if (newState.status) {
+ status = newState.status
+ spinner.tick(`${activity.id} — ${newState.status}`)
+ }
+ },
+ done: () => {
+ const str = status
+ ? `${activity.id} — ${calcElapsedTime(start)} — ${status}`
+ : `${activity.id} — ${calcElapsedTime(start)}`
+ reporter.success(str)
+ spinner.end()
+ },
+ }
+ }
+
+ if (activity.type === `progress`) {
+ const bar = new ProgressBar(
+ ` [:bar] :current/:total :elapsed s :percent ${activity.id}`,
+ {
+ total: 0,
+ width: 30,
+ clear: true,
+ }
+ )
+ return {
+ update: newState => {
+ if (newState.startTime) {
+ start = newState.startTime
+ }
+ if (newState.total) {
+ bar.total = newState.total
+ }
+ if (newState.current) {
+ bar.tick()
+ }
+ },
+ done: () => {
+ reporter.success(
+ `${activity.id} — ${bar.curr}/${bar.total} - ${calcElapsedTime(
+ start
+ )} s`
+ )
+ },
+ }
}
return {
- start: () => {
- spinner.tick(name)
- },
- setStatus: s => {
- status = s
- spinner.tick(`${name} — ${status}`)
- },
- end: () => {
- const str = status
- ? `${name} — ${elapsedTime()} — ${status}`
- : `${name} — ${elapsedTime()}`
- reporter.success(str)
- spinner.end()
- },
+ update: () => {},
+ done: () => {},
}
},
}
diff --git a/packages/gatsby-cli/src/util/calc-elapsed-time.js b/packages/gatsby-cli/src/util/calc-elapsed-time.js
new file mode 100644
index 0000000000000..336ed74eee561
--- /dev/null
+++ b/packages/gatsby-cli/src/util/calc-elapsed-time.js
@@ -0,0 +1,7 @@
+const convertHrtime = require(`convert-hrtime`)
+
+module.exports = startTime => {
+ const elapsed = process.hrtime(startTime)
+
+ return convertHrtime(elapsed)[`seconds`].toFixed(3)
+}
diff --git a/packages/gatsby-plugin-sharp/package.json b/packages/gatsby-plugin-sharp/package.json
index 3af88109b089e..d86d05b08cfa4 100644
--- a/packages/gatsby-plugin-sharp/package.json
+++ b/packages/gatsby-plugin-sharp/package.json
@@ -19,7 +19,7 @@
"mini-svg-data-uri": "^1.0.0",
"potrace": "^2.1.1",
"probe-image-size": "^4.0.0",
- "progress": "^1.1.8",
+ "progress": "^2.0.3",
"semver": "^5.6.0",
"sharp": "^0.22.1",
"svgo": "^1.2.0"
diff --git a/packages/gatsby-plugin-sharp/src/__tests__/utils.js b/packages/gatsby-plugin-sharp/src/__tests__/utils.js
new file mode 100644
index 0000000000000..633988595d216
--- /dev/null
+++ b/packages/gatsby-plugin-sharp/src/__tests__/utils.js
@@ -0,0 +1,36 @@
+jest.mock(`gatsby-cli/lib/reporter`)
+jest.mock(`progress`)
+const { createProgress } = require(`../utils`)
+const reporter = require(`gatsby-cli/lib/reporter`)
+const progress = require(`progress`)
+
+describe(`createProgress`, () => {
+ beforeEach(() => {
+ progress.mockClear()
+ })
+
+ it(`should use createProgress from gatsby-cli when available`, () => {
+ createProgress(`test`, reporter)
+ expect(reporter.createProgress).toBeCalled()
+ expect(progress).not.toBeCalled()
+ })
+
+ it(`should fallback to a local implementation when createProgress does not exists on reporter`, () => {
+ reporter.createProgress = null
+ const bar = createProgress(`test`, reporter)
+ expect(progress).toHaveBeenCalledTimes(1)
+ expect(bar).toHaveProperty(`start`, expect.any(Function))
+ expect(bar).toHaveProperty(`tick`, expect.any(Function))
+ expect(bar).toHaveProperty(`done`, expect.any(Function))
+ expect(bar).toHaveProperty(`total`)
+ })
+
+ it(`should fallback to a local implementation when no reporter is present`, () => {
+ const bar = createProgress(`test`)
+ expect(progress).toHaveBeenCalledTimes(1)
+ expect(bar).toHaveProperty(`start`, expect.any(Function))
+ expect(bar).toHaveProperty(`tick`, expect.any(Function))
+ expect(bar).toHaveProperty(`done`, expect.any(Function))
+ expect(bar).toHaveProperty(`total`)
+ })
+})
diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js
index 83f476f58e3a8..17e992621260f 100644
--- a/packages/gatsby-plugin-sharp/src/index.js
+++ b/packages/gatsby-plugin-sharp/src/index.js
@@ -110,7 +110,8 @@ function queueImageResizing({ file, args = {}, reporter }) {
const finishedPromise = scheduleJob(
job,
boundActionCreators,
- pluginOptions
+ pluginOptions,
+ reporter
).then(() => {
queue.delete(prefixedSrc)
})
diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js
index 21cd7fa223dc4..4e691f4020129 100644
--- a/packages/gatsby-plugin-sharp/src/scheduler.js
+++ b/packages/gatsby-plugin-sharp/src/scheduler.js
@@ -1,8 +1,8 @@
const _ = require(`lodash`)
-const ProgressBar = require(`progress`)
const { existsSync } = require(`fs`)
const queue = require(`async/queue`)
const { processFile } = require(`./process-file`)
+const { createProgress } = require(`./utils`)
const toProcess = {}
let totalJobs = 0
@@ -10,18 +10,20 @@ const q = queue((task, callback) => {
task(callback)
}, 1)
-const bar = new ProgressBar(
- `Generating image thumbnails [:bar] :current/:total :elapsed secs :percent`,
- {
- total: 0,
- width: 30,
+let bar
+// when the queue is empty we stop the progressbar
+q.drain = () => {
+ if (bar) {
+ bar.done()
}
-)
+ totalJobs = 0
+}
exports.scheduleJob = async (
job,
boundActionCreators,
pluginOptions,
+ reporter,
reportStatus = true
) => {
const inputFileKey = job.inputPath.replace(/\./g, `%2E`)
@@ -50,6 +52,10 @@ exports.scheduleJob = async (
deferred.resolve = resolve
deferred.reject = reject
})
+ if (totalJobs === 0) {
+ bar = createProgress(`Generating image thumbnails`, reporter)
+ bar.start()
+ }
totalJobs += 1
@@ -107,6 +113,7 @@ function runJobs(
// We're now processing the file's jobs.
let imagesFinished = 0
+
bar.total = totalJobs
try {
diff --git a/packages/gatsby-plugin-sharp/src/utils.js b/packages/gatsby-plugin-sharp/src/utils.js
new file mode 100644
index 0000000000000..acb49ad99c919
--- /dev/null
+++ b/packages/gatsby-plugin-sharp/src/utils.js
@@ -0,0 +1,28 @@
+const ProgressBar = require(`progress`)
+
+// TODO remove in V3
+export function createProgress(message, reporter) {
+ if (reporter && reporter.createProgress) {
+ return reporter.createProgress(message)
+ }
+
+ const bar = new ProgressBar(
+ ` [:bar] :current/:total :elapsed s :percent ${message}`,
+ {
+ total: 0,
+ width: 30,
+ clear: true,
+ }
+ )
+
+ return {
+ start() {},
+ tick() {
+ bar.tick()
+ },
+ done() {},
+ set total(value) {
+ bar.total = value
+ },
+ }
+}
diff --git a/packages/gatsby-source-contentful/src/download-contentful-assets.js b/packages/gatsby-source-contentful/src/download-contentful-assets.js
index 9695e74f318ec..f185a80e3ad6e 100644
--- a/packages/gatsby-source-contentful/src/download-contentful-assets.js
+++ b/packages/gatsby-source-contentful/src/download-contentful-assets.js
@@ -25,6 +25,7 @@ const downloadContentfulAssets = async gatsbyFunctions => {
store,
cache,
getNodes,
+ reporter,
} = gatsbyFunctions
// Any ContentfulAsset nodes will be downloaded, cached and copied to public/static
@@ -62,6 +63,7 @@ const downloadContentfulAssets = async gatsbyFunctions => {
cache,
createNode,
createNodeId,
+ reporter,
})
if (fileNode) {
diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js
index 20f51b5c9bd5c..b08c1d226e171 100644
--- a/packages/gatsby-source-contentful/src/gatsby-node.js
+++ b/packages/gatsby-source-contentful/src/gatsby-node.js
@@ -222,6 +222,7 @@ exports.sourceNodes = async (
store,
cache,
getNodes,
+ reporter,
})
}
diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
index 8846da95eb2ab..0518d942895d2 100644
--- a/packages/gatsby-source-drupal/src/gatsby-node.js
+++ b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -261,6 +261,7 @@ exports.sourceNodes = async (
createNodeId,
parentNodeId: node.id,
auth,
+ reporter,
})
} catch (err) {
reporter.error(err)
diff --git a/packages/gatsby-source-filesystem/index.d.ts b/packages/gatsby-source-filesystem/index.d.ts
index 89812d4dc49fc..d6f72e353bd0c 100644
--- a/packages/gatsby-source-filesystem/index.d.ts
+++ b/packages/gatsby-source-filesystem/index.d.ts
@@ -33,6 +33,7 @@ export interface CreateRemoteFileNodeArgs {
httpHeaders?: object
ext?: string
name?: string
+ reporter: object
}
export interface FileSystemNode extends Node {
diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json
index 2541e19c2e2da..0be5040ce2aaa 100644
--- a/packages/gatsby-source-filesystem/package.json
+++ b/packages/gatsby-source-filesystem/package.json
@@ -17,7 +17,7 @@
"md5-file": "^3.1.1",
"mime": "^2.2.0",
"pretty-bytes": "^4.0.2",
- "progress": "^1.1.8",
+ "progress": "^2.0.3",
"read-chunk": "^3.0.0",
"valid-url": "^1.0.9",
"xstate": "^3.1.0"
diff --git a/packages/gatsby-source-filesystem/src/__tests__/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/__tests__/create-remote-file-node.js
index 44aa34d2cae53..c2a80324fa0dd 100644
--- a/packages/gatsby-source-filesystem/src/__tests__/create-remote-file-node.js
+++ b/packages/gatsby-source-filesystem/src/__tests__/create-remote-file-node.js
@@ -19,30 +19,31 @@ jest.mock(`got`, () => {
stream: jest.fn(),
}
})
-jest.mock(
- `progress`,
- () =>
- class ProgressBar {
- static total = 0
- static tick = jest.fn(() => (ProgressBar.total -= 1))
-
- total = ProgressBar.total
- tick = ProgressBar.tick
- }
-)
+
+jest.mock(`gatsby-cli/lib/reporter`, () => {
+ return {
+ createProgress: jest.fn(),
+ }
+})
jest.mock(`../create-file-node`, () => {
return {
createFileNode: jest.fn(),
}
})
+const reporter = require(`gatsby-cli/lib/reporter`)
+const progressBar = {
+ start: jest.fn(),
+ total: 0,
+ tick: jest.fn(),
+}
+reporter.createProgress.mockImplementation(() => progressBar)
+
const got = require(`got`)
-const ProgressBar = require(`progress`)
const createRemoteFileNode = require(`../create-remote-file-node`)
const { createFileNode } = require(`../create-file-node`)
beforeEach(() => {
- ProgressBar.total = 0
- ProgressBar.tick.mockClear()
+ progressBar.tick.mockClear()
})
describe(`create-remote-file-node`, () => {
@@ -52,6 +53,7 @@ describe(`create-remote-file-node`, () => {
cache: {},
createNode: jest.fn(),
createNodeId: jest.fn(),
+ reporter,
}
describe(`basic functionality`, () => {
@@ -73,8 +75,8 @@ describe(`create-remote-file-node`, () => {
expect(value).rejects.toMatch(`wrong url: `)
- expect(ProgressBar.total).toBe(0)
- expect(ProgressBar.tick).not.toHaveBeenCalled()
+ expect(progressBar.total).toBe(0)
+ expect(progressBar.tick).not.toHaveBeenCalled()
})
})
})
@@ -141,7 +143,8 @@ describe(`create-remote-file-node`, () => {
it(`invokes ProgressBar tick`, async () => {
await setup()
- expect(ProgressBar.tick).toHaveBeenCalledTimes(1)
+ expect(progressBar.total).toBe(1)
+ expect(progressBar.tick).toHaveBeenCalledTimes(1)
})
describe(`requesting remote image`, () => {
diff --git a/packages/gatsby-source-filesystem/src/__tests__/utils.js b/packages/gatsby-source-filesystem/src/__tests__/utils.js
index 298a7fe8f982a..e679448b2490f 100644
--- a/packages/gatsby-source-filesystem/src/__tests__/utils.js
+++ b/packages/gatsby-source-filesystem/src/__tests__/utils.js
@@ -1,4 +1,13 @@
-const { getRemoteFileExtension, getRemoteFileName, slash } = require(`../utils`)
+jest.mock(`gatsby-cli/lib/reporter`)
+jest.mock(`progress`)
+const {
+ getRemoteFileExtension,
+ getRemoteFileName,
+ createProgress,
+ slash,
+} = require(`../utils`)
+const reporter = require(`gatsby-cli/lib/reporter`)
+const progress = require(`progress`)
describe(`create remote file node`, () => {
it(`can correctly retrieve file name and extensions`, () => {
@@ -23,6 +32,37 @@ describe(`create remote file node`, () => {
})
})
+describe(`createProgress`, () => {
+ beforeEach(() => {
+ progress.mockClear()
+ })
+
+ it(`should use createProgress from gatsby-cli when available`, () => {
+ createProgress(`test`, reporter)
+ expect(reporter.createProgress).toBeCalled()
+ expect(progress).not.toBeCalled()
+ })
+
+ it(`should fallback to a local implementation when createProgress does not exists on reporter`, () => {
+ reporter.createProgress = null
+ const bar = createProgress(`test`, reporter)
+ expect(progress).toHaveBeenCalledTimes(1)
+ expect(bar).toHaveProperty(`start`, expect.any(Function))
+ expect(bar).toHaveProperty(`tick`, expect.any(Function))
+ expect(bar).toHaveProperty(`done`, expect.any(Function))
+ expect(bar).toHaveProperty(`total`)
+ })
+
+ it(`should fallback to a local implementation when no reporter is present`, () => {
+ const bar = createProgress(`test`)
+ expect(progress).toHaveBeenCalledTimes(1)
+ expect(bar).toHaveProperty(`start`, expect.any(Function))
+ expect(bar).toHaveProperty(`tick`, expect.any(Function))
+ expect(bar).toHaveProperty(`done`, expect.any(Function))
+ expect(bar).toHaveProperty(`total`)
+ })
+})
+
describe(`slash path`, () => {
it(`can correctly slash path`, () => {
;[
@@ -34,6 +74,7 @@ describe(`slash path`, () => {
expect(slash(path)).toBe(expectRes)
})
})
+
it(`does not modify extended length paths`, () => {
const extended = `\\\\?\\some\\path`
expect(slash(extended)).toBe(extended)
diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js
index 8471d1dbf4e1c..9cb43ac80ab79 100644
--- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js
+++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js
@@ -6,19 +6,15 @@ const { isWebUri } = require(`valid-url`)
const Queue = require(`better-queue`)
const readChunk = require(`read-chunk`)
const fileType = require(`file-type`)
-const ProgressBar = require(`progress`)
+const { createProgress } = require(`./utils`)
const { createFileNode } = require(`./create-file-node`)
const { getRemoteFileExtension, getRemoteFileName } = require(`./utils`)
const cacheId = url => `create-remote-file-node-${url}`
-const bar = new ProgressBar(
- `Downloading remote files [:bar] :current/:total :elapsed secs :percent`,
- {
- total: 0,
- width: 30,
- }
-)
+let bar
+// Keep track of the total number of jobs we push in the queue
+let totalJobs = 0
/********************
* Type Definitions *
@@ -34,6 +30,11 @@ const bar = new ProgressBar(
* @see gatsby/packages/gatsby/utils/cache.js
*/
+/**
+ * @typedef {Reporter}
+ * @see gatsby/packages/gatsby-cli/lib/reporter.js
+ */
+
/**
* @typedef {Auth}
* @type {Object}
@@ -51,6 +52,7 @@ const bar = new ProgressBar(
* @param {GatsbyCache} options.cache
* @param {Function} options.createNode
* @param {Auth} [options.auth]
+ * @param {Reporter} [options.reporter]
*/
const CACHE_DIR = `.cache`
@@ -84,6 +86,14 @@ const queue = new Queue(pushToQueue, {
concurrent: process.env.GATSBY_CONCURRENT_DOWNLOAD || 200,
})
+// when the queue is empty we stop the progressbar
+queue.on(`drain`, () => {
+ if (bar) {
+ bar.done()
+ }
+ totalJobs = 0
+})
+
/**
* @callback {Queue~queueCallback}
* @param {*} error
@@ -271,9 +281,6 @@ const pushTask = task =>
})
})
-// Keep track of the total number of jobs we push in the queue
-let totalJobs = 0
-
/***************
* Entry Point *
***************/
@@ -300,6 +307,7 @@ module.exports = ({
createNodeId,
ext = null,
name = null,
+ reporter,
}) => {
// validation of the input
// without this it's notoriously easy to pass in the wrong `createNodeId`
@@ -329,6 +337,11 @@ module.exports = ({
return Promise.reject(`wrong url: ${url}`)
}
+ if (totalJobs === 0) {
+ bar = createProgress(`Downloading remote files`, reporter)
+ bar.start()
+ }
+
totalJobs += 1
bar.total = totalJobs
diff --git a/packages/gatsby-source-filesystem/src/utils.js b/packages/gatsby-source-filesystem/src/utils.js
index 24e03f31e525d..b511e5289b57b 100644
--- a/packages/gatsby-source-filesystem/src/utils.js
+++ b/packages/gatsby-source-filesystem/src/utils.js
@@ -1,5 +1,6 @@
const path = require(`path`)
const Url = require(`url`)
+const ProgressBar = require(`progress`)
/**
* getParsedPath
@@ -40,6 +41,33 @@ export function getRemoteFileName(url) {
return getParsedPath(url).name
}
+// TODO remove in V3
+export function createProgress(message, reporter) {
+ if (reporter && reporter.createProgress) {
+ return reporter.createProgress(message)
+ }
+
+ const bar = new ProgressBar(
+ ` [:bar] :current/:total :elapsed s :percent ${message}`,
+ {
+ total: 0,
+ width: 30,
+ clear: true,
+ }
+ )
+
+ return {
+ start() {},
+ tick() {
+ bar.tick()
+ },
+ done() {},
+ set total(value) {
+ bar.total = value
+ },
+ }
+}
+
/**
* slash
* --
diff --git a/packages/gatsby-source-shopify/src/gatsby-node.js b/packages/gatsby-source-shopify/src/gatsby-node.js
index ac01f6c1edccf..556dae7482eb3 100644
--- a/packages/gatsby-source-shopify/src/gatsby-node.js
+++ b/packages/gatsby-source-shopify/src/gatsby-node.js
@@ -25,7 +25,7 @@ import {
} from "./queries"
export const sourceNodes = async (
- { actions: { createNode, touchNode }, createNodeId, store, cache },
+ { actions: { createNode, touchNode }, createNodeId, store, cache, reporter },
{ shopName, accessToken, verbose = true, paginationSize = 250 }
) => {
const client = createClient(shopName, accessToken)
@@ -38,7 +38,14 @@ export const sourceNodes = async (
console.log(formatMsg(`starting to fetch data from Shopify`))
// Arguments used for file node creation.
- const imageArgs = { createNode, createNodeId, touchNode, store, cache }
+ const imageArgs = {
+ createNode,
+ createNodeId,
+ touchNode,
+ store,
+ cache,
+ reporter,
+ }
// Arguments used for node creation.
const args = {
diff --git a/packages/gatsby-source-shopify/src/nodes.js b/packages/gatsby-source-shopify/src/nodes.js
index d8e9f900839bb..5ffd77a4a132a 100644
--- a/packages/gatsby-source-shopify/src/nodes.js
+++ b/packages/gatsby-source-shopify/src/nodes.js
@@ -23,7 +23,7 @@ const { createNodeFactory, generateNodeId } = createNodeHelpers({
const downloadImageAndCreateFileNode = async (
{ url, nodeId },
- { createNode, createNodeId, touchNode, store, cache }
+ { createNode, createNodeId, touchNode, store, cache, reporter }
) => {
let fileNodeID
@@ -43,6 +43,7 @@ const downloadImageAndCreateFileNode = async (
createNode,
createNodeId,
parentNodeId: nodeId,
+ reporter,
})
if (fileNode) {
diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js
index 0cbb279c0597f..da35d8b92df73 100644
--- a/packages/gatsby-source-wordpress/src/gatsby-node.js
+++ b/packages/gatsby-source-wordpress/src/gatsby-node.js
@@ -24,7 +24,15 @@ let _excludedRoutes
let _normalizer
exports.sourceNodes = async (
- { actions, getNode, store, cache, createNodeId, createContentDigest },
+ {
+ actions,
+ getNode,
+ store,
+ cache,
+ createNodeId,
+ createContentDigest,
+ reporter,
+ },
{
baseUrl,
protocol,
@@ -123,6 +131,7 @@ exports.sourceNodes = async (
touchNode,
getNode,
_auth,
+ reporter,
})
// Creates links between elements and parent element.
diff --git a/packages/gatsby-source-wordpress/src/normalize.js b/packages/gatsby-source-wordpress/src/normalize.js
index 0de9e2c4cf1ee..40358a5bb76e4 100644
--- a/packages/gatsby-source-wordpress/src/normalize.js
+++ b/packages/gatsby-source-wordpress/src/normalize.js
@@ -477,6 +477,7 @@ exports.downloadMediaFiles = async ({
touchNode,
getNode,
_auth,
+ reporter,
}) =>
Promise.all(
entities.map(async e => {
@@ -511,6 +512,7 @@ exports.downloadMediaFiles = async ({
createNodeId,
parentNodeId: e.id,
auth: _auth,
+ reporter,
})
if (fileNode) {
diff --git a/packages/gatsby-transformer-screenshot/src/gatsby-node.js b/packages/gatsby-transformer-screenshot/src/gatsby-node.js
index 7f667137a8ed6..e4ffa79535fd2 100644
--- a/packages/gatsby-transformer-screenshot/src/gatsby-node.js
+++ b/packages/gatsby-transformer-screenshot/src/gatsby-node.js
@@ -16,7 +16,15 @@ const screenshotQueue = new Queue(
)
exports.onPreBootstrap = (
- { store, cache, actions, createNodeId, getNodesByType, createContentDigest },
+ {
+ store,
+ cache,
+ actions,
+ createNodeId,
+ getNodesByType,
+ createContentDigest,
+ reporter,
+ },
pluginOptions
) => {
const { createNode, touchNode } = actions
@@ -46,6 +54,7 @@ exports.onPreBootstrap = (
createNodeId,
parentNodeId: n.id,
createContentDigest,
+ reporter,
})
} else {
// Screenshot hasn't yet expired, touch the image node
@@ -119,6 +128,7 @@ const createScreenshotNode = async ({
createNodeId,
parentNodeId,
createContentDigest,
+ reporter,
}) => {
try {
let fileNode, expires
@@ -139,6 +149,7 @@ const createScreenshotNode = async ({
createNode,
createNodeId,
parentNodeId,
+ reporter,
})
expires = screenshotResponse.data.expires
diff --git a/yarn.lock b/yarn.lock
index 4f0d7ddecf269..a2e93a5ba0155 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -17472,16 +17472,16 @@ process@~0.5.1:
resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf"
integrity sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8=
-progress@^1.1.8:
- version "1.1.8"
- resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be"
- integrity sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=
-
progress@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.0.tgz#8a1be366bf8fc23db2bd23f10c6fe920b4389d1f"
integrity sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=
+progress@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
+ integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
+
promise-inflight@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"