Skip to content

Commit

Permalink
Resolves #379: add metalsmith.watch option setter and watcher
Browse files Browse the repository at this point in the history
  • Loading branch information
webketje committed May 27, 2023
1 parent 48a0167 commit 9d40674
Show file tree
Hide file tree
Showing 4 changed files with 272 additions and 58 deletions.
119 changes: 103 additions & 16 deletions lib/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
const assert = require('assert')
const Mode = require('stat-mode')
const path = require('path')
const watcher = require('./watcher')

const {
readdir,
batchAsync,
Expand All @@ -26,7 +28,9 @@ const { Debugger, fileLogHandler } = require('./debug')

const symbol = {
env: Symbol('env'),
log: Symbol('log')
log: Symbol('log'),
watch: Symbol('watch'),
closeWatcher: Symbol('closeWatcher')
}

/**
Expand Down Expand Up @@ -109,6 +113,9 @@ module.exports = Metalsmith
* @return {Metalsmith}
*/

/**
* @constructor
*/
function Metalsmith(directory) {
if (!(this instanceof Metalsmith)) return new Metalsmith(directory)
assert(directory, 'You must pass a working directory path.')
Expand All @@ -131,6 +138,16 @@ function Metalsmith(directory) {
enumerable: false,
writable: true
})
Object.defineProperty(this, symbol.watch, {
value: false,
enumerable: false,
writable: true
})
Object.defineProperty(this, symbol.closeWatcher, {
value: null,
enumerable: false,
writable: true
})
}

/**
Expand Down Expand Up @@ -412,20 +429,22 @@ Metalsmith.prototype.build = function (callback) {
if (this[symbol.log].pending) {
this[symbol.log].on('ready', () => resolve())
} else {
/* istanbul ignore next */
resolve()
}
})
}
})
.then(this.process.bind(this))
.then((files) => {
return this.write(files).then(() => {
if (this[symbol.log]) this[symbol.log].end()
return files
.then(
this.process.bind(this, (err, files) => {
if (err) throw err
return this.write(files)
.then(() => {
if (this[symbol.log]) this[symbol.log].end()
if (isFunction(callback)) callback(null, files)
})
.catch(callback)
})
})

)
/* block required for Metalsmith 2.x callback-flow compat */
if (isFunction(callback)) {
result.then((files) => callback(null, files), callback)
Expand All @@ -434,6 +453,65 @@ Metalsmith.prototype.build = function (callback) {
}
}

/**
* **EXPERIMENTAL — Caution**
* * not to be used with @metalsmith/metadata <= 0.2.0: a bug may trigger an infinite loop
* * not to be used with existing watch plugins
* * metalsmith.process/build are **not awaitable** when watching is enabled.
* Instead of running once at the build's end, callbacks passed to these methods will run on every rebuild.
*
* Set the list of paths to watch and trigger rebuilds on. The watch method will skip files ignored with `metalsmith.ignore()`
* and will do partial (true) or full (false) rebuilds depending on the `metalsmith.clean()` setting.
* It can be used both for rebuilding in-memory with `metalsmith.process` or writing to file system with `metalsmith.build`,
* @method Metalsmith#watch
* @param {boolean|string|string[]} [paths]
* @return {Metalsmith|Promise<void>|boolean|import('chokidar').WatchOptions}
* @example
*
* metalsmith
* .ignore(['wont-be-watched']) // ignored
* .clean(false) // do partial rebuilds
* .watch(true) // watch all files in metalsmith.source()
* .watch(['lib','src']) // or watch files in directories 'lib' and 'src'
*
* if (process.argv[2] === '--dry-run') {
* metalsmith.process(onRebuild) // reprocess in memory without writing to disk
* } else {
* metalsmith.build(onRebuild) // rewrite to disk
* }
*
* function onRebuild(err, files) {
* if (err) {
* metalsmith.watch(false) // stop watching
* .finally(() => console.log(err)) // and log build error
* }
* console.log('reprocessed files', Object.keys(files).join(', ')))
* }
*/
Metalsmith.prototype.watch = function (options) {
if (isUndefined(options)) return this[symbol.watch]
if (!options) {
// if watch has previously been enabled and is now passed false, close the watcher
this[symbol.watch] = false
if (options === false && typeof this[symbol.closeWatcher] === 'function') {
return this[symbol.closeWatcher]()
}
} else {
if (isString(options) || Array.isArray(options)) options = { paths: options }
else if (options === true) options = { paths: this.source() }

this[symbol.watch] = {
paths: options.paths,
alwaysStat: false,
cwd: this.directory(),
ignored: this.ignore(),
ignoreInitial: true,
awaitWriteFinish: true
}
}
return this
}

/**
* Process files through plugins without writing out files.
*
Expand All @@ -450,15 +528,24 @@ Metalsmith.prototype.build = function (callback) {
*/

Metalsmith.prototype.process = function (callback) {
const result = this.read(this.source()).then((files) => {
return this.run(files, this.plugins)
})
const result = this.read(this.source())

/* block required for Metalsmith 2.x callback-flow compat */
if (callback) {
result.then((files) => callback(null, files), callback)
if (this.watch()) {
return result.then((files) => {
const msWatcher = watcher(files, this)
msWatcher(this[symbol.watch], callback).then((close) => {
this[symbol.closeWatcher] = close
})
})
} else {
return result
result.then((files) => this.run(files, this.plugins))

/* block required for Metalsmith 2.x callback-flow compat */
if (callback) {
result.then((files) => callback(null, files), callback)
} else {
return result
}
}
}

Expand Down
151 changes: 151 additions & 0 deletions lib/watcher.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
const chokidar = require('chokidar')
// to be replaced in distant future by native structuredClone when dropping Node <17 support
const cloneDeep = require('lodash.clonedeep')
const crypto = require('crypto')
const { relative } = require('path')
const { rm } = require('./helpers')

function sourceRelPath(p, ms) {
return relative(ms.source(), ms.path(p))
}
function isInSource(p) {
return !p.startsWith('..')
}

function computeHashMap(files) {
return Object.entries(files).reduce((hashes, [path, file]) => {
hashes[path] = crypto.createHash('md5').update(file.contents).digest('hex')
return hashes
}, {})
}

/**
* @type {Object<string, string>} HashMap
*/

/**
* Return the keys of `map1` that are different from `map2`
* @param {HashMap} map1
* @param {HashMap} map2
* @returns {Array}
*/
function diffHashMap(map1, map2) {
return Object.keys(map1).filter((path) => map1[path] !== map2[path])
}

module.exports = function watchable(files, metalsmith) {
const clean = metalsmith.clean()
const meta = metalsmith.metadata()
const fileCache = files
let lastHashmap

function rerun() {
return metalsmith.metadata(meta).run(cloneDeep(fileCache), metalsmith.plugins)
}

function transformFilesObj(evt, p, metalsmith) {
// we only care about in-source files & dirs to update the fileCache
// other files are eventually added or processed by plugins
let filesTransform = Promise.resolve()
const relPath = sourceRelPath(p, metalsmith)

if (isInSource(relPath)) {
switch (evt) {
case 'unlinkDir':
metalsmith.match(relPath, Object.keys(fileCache)).forEach((r) => delete fileCache[r])
break
case 'unlink':
delete fileCache[relPath]
break
case 'add':
case 'change':
filesTransform = metalsmith.readFile(metalsmith.path(p)).then((file) => {
fileCache[relPath] = file
})
break
}
}

return filesTransform
}

return function watcher({ paths, ...options }, onRebuild) {
const watcher = chokidar.watch(paths || metalsmith.source(), options)

const eventqueue = []
// eslint-disable-next-line no-unused-vars
let inTheMiddleOfABuild = false
let run

watcher.on('all', (evt, p) => {
// eslint-disable-next-line no-console
console.log(evt, p)

// the metalsmith Files object does not output empty dirs,
// wait for the file add/change events instead
if (evt === 'addDir') return

eventqueue.push([evt, p])

clearTimeout(run)
run = setTimeout(() => {
inTheMiddleOfABuild = true
const fileTransforms = Promise.all(eventqueue.map(([evt, p]) => transformFilesObj(evt, p, metalsmith)))

fileTransforms.then(() => {
eventqueue.splice(0, eventqueue.length)
const latestRun = rerun()

if (clean) {
latestRun
.then(
(files) => onRebuild(null, files),
(err) => onRebuild(err)
)
.finally(() => {
inTheMiddleOfABuild = false
})
return
}

latestRun.then(
(files) => {
const newHashMap = computeHashMap(files)
const changedOrRemoved = diffHashMap(lastHashmap, newHashMap)
const addedFiles = diffHashMap(newHashMap, lastHashmap).filter((p) => !changedOrRemoved.includes(p))
const removedFiles = changedOrRemoved.filter((f) => !files[f])
const changedFiles = changedOrRemoved.filter((f) => !!files[f])
const output = [...addedFiles, ...changedFiles].reduce((all, current) => {
all[current] = files[current]
return all
}, {})
lastHashmap = newHashMap
// eslint-disable-next-line no-console
console.log({ addedFiles, removedFiles, changedFiles })

Promise.all(removedFiles.map((f) => rm(f)))
.then(() => onRebuild(null, output), onRebuild)
.finally(() => {
inTheMiddleOfABuild = false
})
},
(err) => onRebuild(err)
)
})
}, 1000)
})
return new Promise((resolve, reject) => {
rerun()
.then((files) => {
if (!clean) lastHashmap = computeHashMap(files)
watcher.on('ready', () => {
onRebuild(null, files)
resolve(function closeWatcher() {
return watcher.unwatch(paths).close()
})
})
})
.catch(reject)
})
}
}
Loading

0 comments on commit 9d40674

Please sign in to comment.