filespecs: Implement new file: specifier behavior

PR-URL: npm/npm#15900
Credit: @iarna

actions: Allow actions to return promises

extract: Eliminate extra callback

finalize: Rewrite finalize as promises

update-linked: Remove update-linked action

install: Remove obsolete invalid action filtering

fetch-package-metadata: Error on installing windows paths on non-windows systems

fetch-package-metadata: Read in modules installed inside of new links

finalize: Act on realpaths because we may be installing inside a link prior to the symlink being made

finalize: Create symlinks of directory deps

deps: Resolve ambiguity for file specifiers in the traditional way

deps: Set link and realpath properties for directory deps

inflate-bundled: Distinguish published bundles from lined modules

node: Add new fromLink attribute to track sourced-to-symlinks deps

diff-trees: Don't try to install deps that are already inside a link

fetch-package-metadata: Improve error messages for link failures

install: Run preinstall lifecycle after finalize

finalize: Start reading package.json files here

decompose-actions: Don't fetch or extract links

deps: Determine if a link matches spec based on where it points

deps: Compute correct save specifier for file spec

deps: Set the isInLink property on new children

deps: When finding the install location, don't walk up out of a symlink unless PRESERVE_SYMLINKS is on

diff-trees: Stop setting isInLink, this is now a first class property

diff-trees: Only exclude children of links from adding if they were already there behind the symlink

inflate-bundled: We are using isInLink now not fromLink

inflate-bundled: realpaths should be built on realpaths

node: Eliminate fromLink as a thing

node: Fill in values for inLink, isInLink & fromBundle

save: When updating a lock/shrinkwrap don't read the damn tree again

shrinkwrap: Fill in version per the new shrinkwrap spec

install: Make _inBundle purely a debugging artifact

inflate-shrinkwrap: Fix how bundle deps are inflated

save: Detect dependency type when saving

recalculateMetadata: Rewrite as simpler and synchronous

prune: Work off computed metadata

refreshPackageJson: Copy all of the in memory version over the on-disk version except empty items

install: Reduce normalize tree function to minimum

realize-package-specifier: Bring into closer alignment with spec

read-shrinkwrap: Use child.isTop not !child.parent

copy-tree: Don't crash of requires or requiredBy are missing

copy-tree: Initialize copied nodes

dedupe: use getRequested instead of child.package._requested

install: recompute the pkg relationships after finalize

Neccessary to align behavior with shrinkwraps where relationship information
isn't available until we read the module off disk.

install: Prune the tree after building it

install: Only save deps if we made changes

inflate-shrinkwrap: Better metadata while inflating

lib/dedupe.js

@@ -13,13 +13,13 @@ var earliestInstallable = require('./install/deps.js').earliestInstallable
 var checkPermissions = require('./install/check-permissions.js')
 var decomposeActions = require('./install/decompose-actions.js')
 var loadExtraneous = require('./install/deps.js').loadExtraneous
-var filterInvalidActions = require('./install/filter-invalid-actions.js')
-var recalculateMetadata = require('./install/deps.js').recalculateMetadata
+var computeMetadata = require('./install/deps.js').computeMetadata
 var sortActions = require('./install/diff-trees.js').sortActions
 var moduleName = require('./utils/module-name.js')
 var packageId = require('./utils/package-id.js')
 var childPath = require('./utils/child-path.js')
 var usage = require('./utils/usage')
+var getRequested = require('./install/get-requested.js')
 
 module.exports = dedupe
 module.exports.Deduper = Deduper
@@ -65,10 +65,16 @@ Deduper.prototype.loadIdealTree = function (cb) {
     } ],
     [this, this.finishTracker, 'loadAllDepsIntoIdealTree'],
 
-    [this, function (next) { recalculateMetadata(this.idealTree, log, next) }]
+    [this, andComputeMetadata(this.idealTree)]
   ], cb)
 }
 
+function andComputeMetadata (tree) {
+  return function (next) {
+    next(null, computeMetadata(tree))
+  }
+}
+
 Deduper.prototype.generateActionsToTake = function (cb) {
   validate('F', arguments)
   log.silly('dedupe', 'generateActionsToTake')
@@ -82,7 +88,6 @@ Deduper.prototype.generateActionsToTake = function (cb) {
       next()
     }],
     [this, this.finishTracker, 'sort-actions'],
-    [filterInvalidActions, this.where, this.differences],
     [checkPermissions, this.differences],
     [decomposeActions, this.differences, this.todo]
   ], cb)
@@ -130,18 +135,18 @@ function hoistChildren_ (tree, diff, seen, next) {
   seen[tree.path] = true
   asyncMap(tree.children, function (child, done) {
     if (!tree.parent) return hoistChildren_(child, diff, seen, done)
-    var better = findRequirement(tree.parent, moduleName(child), child.package._requested || npa(packageId(child)))
+    var better = findRequirement(tree.parent, moduleName(child), getRequested(child) || npa(packageId(child)))
     if (better) {
       return chain([
         [remove, child, diff],
-        [recalculateMetadata, tree, log]
+        [andComputeMetadata(tree)]
       ], done)
     }
     var hoistTo = earliestInstallable(tree, tree.parent, child.package)
     if (hoistTo) {
       move(child, hoistTo, diff)
       chain([
-        [recalculateMetadata, hoistTo, log],
+        [andComputeMetadata(hoistTo)],
         [hoistChildren_, child, diff, seen],
         [ function (next) {
           moveRemainingChildren(child, diff)

lib/fetch-package-metadata.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const deprCheck = require('./utils/depr-check')
+const path = require('path')
 const log = require('npmlog')
 const readPackageTree = require('read-package-tree')
 const rimraf = require('rimraf')
@@ -12,12 +13,13 @@ const limit = require('call-limit')
 const tempFilename = require('./utils/temp-filename')
 const pacote = require('pacote')
 const pacoteOpts = require('./config/pacote')
+const isWindows = require('./utils/is-windows.js')
 
 function andLogAndFinish (spec, tracker, done) {
-  validate('SF', [spec, done])
+  validate('SOF|SZF|OOF|OZF', [spec, tracker, done])
   return (er, pkg) => {
     if (er) {
-      log.silly('fetchPackageMetaData', 'error for ' + spec, er)
+      log.silly('fetchPackageMetaData', 'error for ' + String(spec), er)
       if (tracker) tracker.finish()
     }
     return done(er, pkg)
@@ -38,28 +40,60 @@ function fetchPackageMetadata (spec, where, opts, done) {
     opts = {}
   }
   var tracker = opts.tracker
+  const logAndFinish = andLogAndFinish(spec, tracker, done)
+
   if (typeof spec === 'object') {
     var dep = spec
-    spec = dep.raw
+  } else {
+    dep = npa(spec)
   }
-  const logAndFinish = andLogAndFinish(spec, tracker, done)
-  pacote.manifest(dep || spec, pacoteOpts({
+  if (!isWindows && dep.type === 'directory' && /^[a-zA-Z]:/.test(dep.fetchSpec)) {
+    var err = new Error(`Can't install from windows path on a non-windows system: ${dep.fetchSpec.replace(/[/]/g, '\\')}`)
+    err.code = 'EWINDOWSPATH'
+    return logAndFinish(err)
+  }
+
+  pacote.manifest(dep, pacoteOpts({
     annotate: true,
     fullMetadata: opts.fullMetadata,
     log: tracker || npmlog,
     memoize: CACHE,
     where: where
   })).then(
     (pkg) => logAndFinish(null, deprCheck(pkg)),
-    logAndFinish
+    (err) => {
+      if (dep.type !== 'directory') return logAndFinish(err)
+      if (err.code === 'ENOTDIR') {
+        var enolocal = new Error(`Could not install "${path.relative(process.cwd(), dep.fetchSpec)}" as it is not a directory and is not a file with a name ending in .tgz, .tar.gz or .tar`)
+        enolocal.code = 'ENOLOCAL'
+        if (err.stack) enolocal.stack = err.stack
+        return logAndFinish(enolocal)
+      } else if (err.code === 'ENOPACKAGEJSON') {
+        var enopackage = new Error(`Could not install from "${path.relative(process.cwd(), dep.fetchSpec)}" as it does not contain a package.json file.`)
+        enopackage.code = 'ENOLOCAL'
+        if (err.stack) enopackage.stack = err.stack
+        return logAndFinish(enopackage)
+      } else {
+        return logAndFinish(err)
+      }
+    }
   )
 }
 
 module.exports.addBundled = addBundled
 function addBundled (pkg, next) {
   validate('OF', arguments)
   if (pkg._bundled !== undefined) return next(null, pkg)
-  if (!pkg.bundleDependencies) return next(null, pkg)
+
+  if (!pkg.bundleDependencies && pkg._requested.type !== 'directory') return next(null, pkg)
+  const requested = pkg._requested || npa(pkg._from)
+  if (requested.type === 'directory') {
+    pkg._bundled = null
+    return readPackageTree(pkg._requested.fetchSpec, function (er, tree) {
+      if (tree) pkg._bundled = tree.children
+      return next(null, pkg)
+    })
+  }
   pkg._bundled = null
   const target = tempFilename('unpack')
   const opts = pacoteOpts({integrity: pkg._integrity})

lib/install.js

@@ -118,7 +118,7 @@ var saveMetrics = require('./utils/metrics.js').save
 // install specific libraries
 var copyTree = require('./install/copy-tree.js')
 var readShrinkwrap = require('./install/read-shrinkwrap.js')
-var recalculateMetadata = require('./install/deps.js').recalculateMetadata
+var computeMetadata = require('./install/deps.js').computeMetadata
 var prefetchDeps = require('./install/deps.js').prefetchDeps
 var loadDeps = require('./install/deps.js').loadDeps
 var loadDevDeps = require('./install/deps.js').loadDevDeps
@@ -128,7 +128,6 @@ var loadExtraneous = require('./install/deps.js').loadExtraneous
 var diffTrees = require('./install/diff-trees.js')
 var checkPermissions = require('./install/check-permissions.js')
 var decomposeActions = require('./install/decompose-actions.js')
-var filterInvalidActions = require('./install/filter-invalid-actions.js')
 var validateTree = require('./install/validate-tree.js')
 var validateArgs = require('./install/validate-args.js')
 var saveRequested = require('./install/save.js').saveRequested
@@ -141,6 +140,8 @@ var removeObsoleteDep = require('./install/deps.js').removeObsoleteDep
 var packageId = require('./utils/package-id.js')
 var moduleName = require('./utils/module-name.js')
 var errorMessage = require('./utils/error-message.js')
+var removeDeps = require('./install/deps.js').removeDeps
+var isExtraneous = require('./install/is-extraneous.js')
 
 function unlockCB (lockPath, name, cb) {
   validate('SSF', arguments)
@@ -281,8 +282,12 @@ Installer.prototype.run = function (_cb) {
       [this, this.runPostinstallTopLevelLifecycles],
       [this, this.finishTracker, 'runTopLevelLifecycles']
     )
-    if (getSaveType(this.args)) {
-      postInstallSteps.push([this, this.saveToDependencies])
+    if (getSaveType()) {
+      postInstallSteps.push(
+        // this is necessary as we don't fill in `dependencies` and `devDependencies` in deps loaded from shrinkwrap
+        // until after we extract them
+        [this, (next) => { computeMetadata(this.idealTree); next() }],
+        [this, this.saveToDependencies])
     }
   }
   postInstallSteps.push(
@@ -340,11 +345,24 @@ Installer.prototype.loadCurrentTree = function (cb) {
   } else {
     todo.push([this, this.readLocalPackageData])
   }
-  todo.push(
-    [this, this.normalizeTree, log.newGroup('loadCurrentTree:normalizeTree')])
+  todo.push([this, this.normalizeCurrentTree])
   chain(todo, cb)
 }
 
+var createNode = require('./install/node.js').create
+var flatNameFromTree = require('./install/flatten-tree.js').flatNameFromTree
+Installer.prototype.normalizeCurrentTree = function (cb) {
+  this.currentTree.isTop = true
+  normalizeTree(this.currentTree)
+  return cb()
+
+  function normalizeTree (tree) {
+    createNode(tree)
+    tree.location = flatNameFromTree(tree)
+    tree.children.forEach(normalizeTree)
+  }
+}
+
 Installer.prototype.loadIdealTree = function (cb) {
   validate('F', arguments)
   log.silly('install', 'loadIdealTree')
@@ -361,16 +379,22 @@ Installer.prototype.loadIdealTree = function (cb) {
     [this.newTracker(this.progress.loadIdealTree, 'loadIdealTree:loadAllDepsIntoIdealTree', 10)],
     [this, this.loadAllDepsIntoIdealTree],
     [this, this.finishTracker, 'loadIdealTree:loadAllDepsIntoIdealTree'],
-
-    // TODO: Remove this (should no longer be necessary, instead counter productive)
-    [this, function (next) { recalculateMetadata(this.idealTree, log, next) }]
+    [this, function (next) { computeMetadata(this.idealTree); next() }],
+    [this, this.pruneIdealTree]
   ], cb)
 }
 
+Installer.prototype.pruneIdealTree = function (cb) {
+  var toPrune = this.idealTree.children
+    .filter((n) => !n.fromShrinkwrap && isExtraneous(n))
+    .map((n) => ({name: moduleName(n)}))
+  return removeDeps(toPrune, this.idealTree, null, log.newGroup('pruneDeps'), cb)
+}
+
 Installer.prototype.loadAllDepsIntoIdealTree = function (cb) {
   validate('F', arguments)
   log.silly('install', 'loadAllDepsIntoIdealTree')
-  var saveDeps = getSaveType(this.args)
+  var saveDeps = getSaveType()
 
   var cg = this.progress['loadIdealTree:loadAllDepsIntoIdealTree']
   var installNewModules = !!this.args.length
@@ -386,8 +410,7 @@ Installer.prototype.loadAllDepsIntoIdealTree = function (cb) {
     )
     if (this.prod || this.dev) {
       steps.push(
-        [prefetchDeps, this.idealTree, depsToPreload, cg.newGroup('prefetchDeps')]
-      )
+        [prefetchDeps, this.idealTree, depsToPreload, cg.newGroup('prefetchDeps')])
     }
     if (this.prod) {
       steps.push(
@@ -411,7 +434,6 @@ Installer.prototype.generateActionsToTake = function (cb) {
     [validateTree, this.idealTree, cg.newGroup('validateTree')],
     [diffTrees, this.currentTree, this.idealTree, this.differences, cg.newGroup('diffTrees')],
     [this, this.computeLinked],
-    [filterInvalidActions, this.where, this.differences],
     [checkPermissions, this.differences],
     [decomposeActions, this.differences, this.todo]
   ], cb)
@@ -474,10 +496,11 @@ Installer.prototype.executeActions = function (cb) {
     [lock, node_modules, '.staging'],
     [rimraf, staging],
     [doParallelActions, 'extract', staging, todo, cg.newGroup('extract', 100)],
-    [doParallelActions, 'preinstall', staging, todo, trackLifecycle.newGroup('preinstall')],
     [doReverseSerialActions, 'remove', staging, todo, cg.newGroup('remove')],
     [doSerialActions, 'move', staging, todo, cg.newGroup('move')],
     [doSerialActions, 'finalize', staging, todo, cg.newGroup('finalize')],
+    [doParallelActions, 'refresh-package-json', staging, todo, cg.newGroup('refresh-package-json')],
+    [doParallelActions, 'preinstall', staging, todo, trackLifecycle.newGroup('preinstall')],
     [doSerialActions, 'build', staging, todo, trackLifecycle.newGroup('build')],
     [doSerialActions, 'global-link', staging, todo, trackLifecycle.newGroup('global-link')],
     [doParallelActions, 'update-linked', staging, todo, trackLifecycle.newGroup('update-linked')],
@@ -557,6 +580,7 @@ Installer.prototype.runPostinstallTopLevelLifecycles = function (cb) {
 Installer.prototype.saveToDependencies = function (cb) {
   validate('F', arguments)
   if (this.failing) return cb()
+  if (!this.differences.length) return cb()
   log.silly('install', 'saveToDependencies')
   saveRequested(this.args, this.idealTree, cb)
 }
@@ -623,19 +647,6 @@ Installer.prototype.loadShrinkwrap = function (cb) {
   readShrinkwrap.andInflate(this.idealTree, cb)
 }
 
-Installer.prototype.normalizeTree = function (log, cb) {
-  validate('OF', arguments)
-  log.silly('install', 'normalizeTree')
-  recalculateMetadata(this.currentTree, log, iferr(cb, function (tree) {
-    tree.children.forEach(function (child) {
-      if (child.requiredBy.length === 0) {
-        child.existing = true
-      }
-    })
-    cb(null, tree)
-  }))
-}
-
 Installer.prototype.getInstalledModules = function () {
   return this.differences.filter(function (action) {
     var mutation = action[0]

lib/install/action/extract.js

@@ -15,10 +15,9 @@ const pacote = require('pacote')
 const pacoteOpts = require('../../config/pacote')
 const path = require('path')
 const readJson = BB.promisify(require('read-package-json'))
-const updatePackageJson = BB.promisify(require('../update-package-json'))
 
 module.exports = extract
-function extract (staging, pkg, log, next) {
+function extract (staging, pkg, log) {
   log.silly('extract', packageId(pkg))
   const up = npm.config.get('unsafe-perm')
   const user = up ? null : npm.config.get('user')
@@ -49,14 +48,13 @@ function extract (staging, pkg, log, next) {
       delete metadata.readmeFilename
       pkg.package = metadata
     }
-    return updatePackageJson(pkg, extractTo)
   }).then(() => {
     if (pkg.package.bundleDependencies) {
       return readBundled(pkg, staging, extractTo)
     }
   }).then(() => {
     return gentlyRm(path.join(extractTo, 'node_modules'))
-  }).then(() => next(), next)
+  })
 }
 
 function readBundled (pkg, staging, extractTo) {
@@ -101,8 +99,6 @@ function finishModule (bundler, child, stageTo, stageFrom) {
   if (child.fromBundle === bundler) {
     return mkdirp(path.dirname(stageTo)).then(() => {
       return move(stageFrom, stageTo)
-    }).then(() => {
-      return updatePackageJson(child, stageTo)
     })
   } else {
     return fs.statAsync(stageFrom).then(() => {