This repository has been archived by the owner on Jan 7, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that fs.realpath can (and should!) do. The only thing that's relevant to reading a package tree is whether package folders are symbolic links, and if so, where they point. Additionally, we don't need to re-start the fs.lstat party every time we walk to a new directory. While it makes sense for fs.realpath to do this in the general case, it's not required when reading a package tree, and results in a geometric explosion of lstat syscalls. For example, if a project is in /Users/hyooman/projects/company/website, and it has 1000 dependencies in node_modules, then a whopping 6,000 lstat calls will be made just to repeatedly verify that /Users/hyooman/projects/company/website/node_modules has not moved! In this implementation, every realpath call is cached, as is every lstat. Additionally, process.cwd() is assumed to be "real enough", and added to the cache initially, which means almost never having to walk all the way up to the root directory. In the npm cli project, this drops the lstat count from 14885 to 3054 for a single call to read-package-tree on my system. Larger projects, or projects deeper in a folder tree, will have even larger reductions. This does not account, itself, for a particularly large speed-up, since lstat calls do tend to be fairly fast, and the repetitiveness means that there are a lot of hits in the file system's stat cache. But it does make read-package-tree 10-30% faster in common use cases.
- Loading branch information
Showing
5 changed files
with
238 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| // look up the realpath, but cache stats to minimize overhead | ||
| // If the parent folder is in the realpath cache, then we just | ||
| // lstat the child, since there's no need to do a full realpath | ||
| // This is not a separate module, and is much simpler than Node's | ||
| // built-in fs.realpath, because we only care about symbolic links, | ||
| // so we can handle many fewer edge cases. | ||
|
|
||
| const fs = require('fs') | ||
| const { promisify } = require('util') | ||
| const readlink = promisify(fs.readlink) | ||
| const lstat = promisify(fs.lstat) | ||
| const { resolve, basename, dirname } = require('path') | ||
|
|
||
| const realpathCached = (path, rpcache, stcache, depth) => { | ||
| // just a safety against extremely deep eloops | ||
| /* istanbul ignore next */ | ||
| if (depth > 2000) | ||
| throw eloop(path) | ||
|
|
||
| if (rpcache.has(path)) | ||
| return Promise.resolve(rpcache.get(path)) | ||
|
|
||
| const dir = dirname(path) | ||
| const base = basename(path) | ||
|
|
||
| if (base && rpcache.has(dir)) | ||
| return realpathChild(dir, base, rpcache, stcache, depth) | ||
|
|
||
| // if it's the root, then we know it's real | ||
| if (!base) { | ||
| rpcache.set(dir, dir) | ||
| return Promise.resolve(dir) | ||
| } | ||
|
|
||
| // the parent, what is that? | ||
| // find out, and then come back. | ||
| return realpathCached(dir, rpcache, stcache, depth + 1).then(() => | ||
| realpathCached(path, rpcache, stcache, depth + 1)) | ||
| } | ||
|
|
||
| const lstatCached = (path, stcache) => { | ||
| if (stcache.has(path)) | ||
| return Promise.resolve(stcache.get(path)) | ||
|
|
||
| const p = lstat(path).then(st => { | ||
| stcache.set(path, st) | ||
| return st | ||
| }) | ||
| stcache.set(path, p) | ||
| return p | ||
| } | ||
|
|
||
| // This is a slight fib, as it doesn't actually occur during a stat syscall. | ||
| // But file systems are giant piles of lies, so whatever. | ||
| const eloop = path => | ||
| Object.assign(new Error( | ||
| `ELOOP: too many symbolic links encountered, stat '${path}'`), { | ||
| errno: -62, | ||
| syscall: 'stat', | ||
| code: 'ELOOP', | ||
| path: path, | ||
| }) | ||
|
|
||
| const realpathChild = (dir, base, rpcache, stcache, depth) => { | ||
| const realdir = rpcache.get(dir) | ||
| // that unpossible | ||
| /* istanbul ignore next */ | ||
| if (typeof realdir === 'undefined') | ||
| throw new Error('in realpathChild without parent being in realpath cache') | ||
|
|
||
| const realish = resolve(realdir, base) | ||
| return lstatCached(realish, stcache).then(st => { | ||
| if (!st.isSymbolicLink()) { | ||
| rpcache.set(resolve(dir, base), realish) | ||
| return realish | ||
| } | ||
|
|
||
| let res | ||
| return readlink(realish).then(target => { | ||
| const resolved = res = resolve(realdir, target) | ||
| if (realish === resolved) | ||
| throw eloop(realish) | ||
|
|
||
| return realpathCached(resolved, rpcache, stcache, depth + 1) | ||
| }).then(real => { | ||
| rpcache.set(resolve(dir, base), real) | ||
| return real | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| module.exports = realpathCached |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters