diff --git a/lib/cache.js b/lib/cache.js index 169f192cad5f2..00abd8c746ab7 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -9,9 +9,9 @@ const finished = BB.promisify(require('mississippi').finished) const log = require('npmlog') const npa = require('npm-package-arg') const npm = require('./npm.js') +const npmConfig = require('./config/figgy-config.js') const output = require('./utils/output.js') const pacote = require('pacote') -const pacoteOpts = require('./config/pacote') const path = require('path') const rm = BB.promisify(require('./utils/gently-rm.js')) const unbuild = BB.promisify(npm.commands.unbuild) @@ -107,7 +107,7 @@ function add (args, where) { log.verbose('cache add', 'spec', spec) if (!spec) return BB.reject(new Error(usage)) log.silly('cache add', 'parsed spec', spec) - return finished(pacote.tarball.stream(spec, pacoteOpts({where})).resume()) + return finished(pacote.tarball.stream(spec, npmConfig({where})).resume()) } cache.verify = verify @@ -131,7 +131,7 @@ function verify () { cache.unpack = unpack function unpack (pkg, ver, unpackTarget, dmode, fmode, uid, gid) { return unbuild([unpackTarget], true).then(() => { - const opts = pacoteOpts({dmode, fmode, uid, gid, offline: true}) + const opts = npmConfig({dmode, fmode, uid, gid, offline: true}) return pacote.extract(npa.resolve(pkg, ver), unpackTarget, opts) }) } diff --git a/lib/ci.js b/lib/ci.js index 03822b9528d1d..1fbb28b570f6f 100644 --- a/lib/ci.js +++ b/lib/ci.js @@ -1,40 +1,19 @@ 'use strict' const Installer = require('libcipm') -const lifecycleOpts = require('./config/lifecycle.js') -const npm = require('./npm.js') +const npmConfig = require('./config/figgy-config.js') const npmlog = require('npmlog') -const pacoteOpts = require('./config/pacote.js') ci.usage = 'npm ci' ci.completion = (cb) => cb(null, []) -Installer.CipmConfig.impl(npm.config, { - get: npm.config.get, - set: npm.config.set, - toLifecycle (moreOpts) { - return lifecycleOpts(moreOpts) - }, - toPacote (moreOpts) { - return pacoteOpts(moreOpts) - } -}) - module.exports = ci function ci (args, cb) { - return new Installer({ - config: npm.config, - log: npmlog - }) - .run() - .then( - (details) => { - npmlog.disableProgress() - console.log(`added ${details.pkgCount} packages in ${ - details.runTime / 1000 - }s`) - } - ) - .then(() => cb(), cb) + return new Installer(npmConfig({ log: npmlog })).run().then(details => { + npmlog.disableProgress() + console.log(`added ${details.pkgCount} packages in ${ + details.runTime / 1000 + }s`) + }).then(() => cb(), cb) } diff --git a/lib/config/figgy-config.js b/lib/config/figgy-config.js index 9f99db329be4b..2da085f40673d 100644 --- a/lib/config/figgy-config.js +++ b/lib/config/figgy-config.js @@ -33,6 +33,7 @@ function mkConfig (...providers) { dirPacker: pack.packGitDep, hashAlgorithm: 'sha1', includeDeprecated: false, + log, 'npm-session': npmSession, 'project-scope': npm.projectScope, refer: npm.refer, diff --git a/lib/config/pacote.js b/lib/config/pacote.js deleted file mode 100644 index 1131c8a055fa2..0000000000000 --- a/lib/config/pacote.js +++ /dev/null @@ -1,142 +0,0 @@ -'use strict' - -const Buffer = require('safe-buffer').Buffer - -const crypto = require('crypto') -const npm = require('../npm') -const log = require('npmlog') -let pack -const path = require('path') - -let effectiveOwner - -const npmSession = crypto.randomBytes(8).toString('hex') -log.verbose('npm-session', npmSession) - -module.exports = pacoteOpts -function pacoteOpts (moreOpts) { - if (!pack) { - pack = require('../pack.js') - } - const ownerStats = calculateOwner() - const opts = { - cache: path.join(npm.config.get('cache'), '_cacache'), - ca: npm.config.get('ca'), - cert: npm.config.get('cert'), - defaultTag: npm.config.get('tag'), - dirPacker: pack.packGitDep, - git: npm.config.get('git'), - hashAlgorithm: 'sha1', - includeDeprecated: false, - key: npm.config.get('key'), - localAddress: npm.config.get('local-address'), - log: log, - maxAge: npm.config.get('cache-min'), - maxSockets: npm.config.get('maxsockets'), - npmSession: npmSession, - offline: npm.config.get('offline'), - preferOffline: npm.config.get('prefer-offline') || npm.config.get('cache-min') > 9999, - preferOnline: npm.config.get('prefer-online') || npm.config.get('cache-max') <= 0, - projectScope: npm.projectScope, - proxy: npm.config.get('https-proxy') || npm.config.get('proxy'), - noProxy: npm.config.get('noproxy'), - refer: npm.referer, - registry: npm.config.get('registry'), - retry: { - retries: npm.config.get('fetch-retries'), - factor: npm.config.get('fetch-retry-factor'), - minTimeout: npm.config.get('fetch-retry-mintimeout'), - maxTimeout: npm.config.get('fetch-retry-maxtimeout') - }, - scope: npm.config.get('scope'), - strictSSL: npm.config.get('strict-ssl'), - userAgent: npm.config.get('user-agent'), - - dmode: npm.modes.exec, - fmode: npm.modes.file, - umask: npm.modes.umask - } - - if (ownerStats.uid != null || ownerStats.gid != null) { - Object.assign(opts, ownerStats) - } - - npm.config.keys.forEach(function (k) { - const authMatchGlobal = k.match( - /^(_authToken|username|_password|password|email|always-auth|_auth)$/ - ) - const authMatchScoped = k[0] === '/' && k.match( - /(.*):(_authToken|username|_password|password|email|always-auth|_auth)$/ - ) - - // if it matches scoped it will also match global - if (authMatchGlobal || authMatchScoped) { - let nerfDart = null - let key = null - let val = null - - if (!opts.auth) { opts.auth = {} } - - if (authMatchScoped) { - nerfDart = authMatchScoped[1] - key = authMatchScoped[2] - val = npm.config.get(k) - if (!opts.auth[nerfDart]) { - opts.auth[nerfDart] = { - alwaysAuth: !!npm.config.get('always-auth') - } - } - } else { - key = authMatchGlobal[1] - val = npm.config.get(k) - opts.auth.alwaysAuth = !!npm.config.get('always-auth') - } - - const auth = authMatchScoped ? opts.auth[nerfDart] : opts.auth - if (key === '_authToken') { - auth.token = val - } else if (key.match(/password$/i)) { - auth.password = - // the config file stores password auth already-encoded. pacote expects - // the actual username/password pair. - Buffer.from(val, 'base64').toString('utf8') - } else if (key === 'always-auth') { - auth.alwaysAuth = val === 'false' ? false : !!val - } else { - auth[key] = val - } - } - - if (k[0] === '@') { - if (!opts.scopeTargets) { opts.scopeTargets = {} } - opts.scopeTargets[k.replace(/:registry$/, '')] = npm.config.get(k) - } - }) - - Object.keys(moreOpts || {}).forEach((k) => { - opts[k] = moreOpts[k] - }) - - return opts -} - -function calculateOwner () { - if (!effectiveOwner) { - effectiveOwner = { uid: 0, gid: 0 } - - // Pretty much only on windows - if (!process.getuid) { - return effectiveOwner - } - - effectiveOwner.uid = +process.getuid() - effectiveOwner.gid = +process.getgid() - - if (effectiveOwner.uid === 0) { - if (process.env.SUDO_UID) effectiveOwner.uid = +process.env.SUDO_UID - if (process.env.SUDO_GID) effectiveOwner.gid = +process.env.SUDO_GID - } - } - - return effectiveOwner -} diff --git a/lib/fetch-package-metadata.js b/lib/fetch-package-metadata.js index cca6dc64f4168..69ae78e51e3f2 100644 --- a/lib/fetch-package-metadata.js +++ b/lib/fetch-package-metadata.js @@ -8,11 +8,11 @@ const rimraf = require('rimraf') const validate = require('aproba') const npa = require('npm-package-arg') const npm = require('./npm') +const npmConfig = require('./config/figgy-config.js') const npmlog = require('npmlog') const limit = require('call-limit') const tempFilename = require('./utils/temp-filename') const pacote = require('pacote') -let pacoteOpts const isWindows = require('./utils/is-windows.js') function andLogAndFinish (spec, tracker, done) { @@ -52,10 +52,7 @@ function fetchPackageMetadata (spec, where, opts, done) { err.code = 'EWINDOWSPATH' return logAndFinish(err) } - if (!pacoteOpts) { - pacoteOpts = require('./config/pacote') - } - pacote.manifest(dep, pacoteOpts({ + pacote.manifest(dep, npmConfig({ annotate: true, fullMetadata: opts.fullMetadata, log: tracker || npmlog, @@ -85,9 +82,6 @@ function fetchPackageMetadata (spec, where, opts, done) { module.exports.addBundled = addBundled function addBundled (pkg, next) { validate('OF', arguments) - if (!pacoteOpts) { - pacoteOpts = require('./config/pacote') - } if (pkg._bundled !== undefined) return next(null, pkg) if (!pkg.bundleDependencies && pkg._requested.type !== 'directory') return next(null, pkg) @@ -101,7 +95,7 @@ function addBundled (pkg, next) { } pkg._bundled = null const target = tempFilename('unpack') - const opts = pacoteOpts({integrity: pkg._integrity}) + const opts = npmConfig({integrity: pkg._integrity}) pacote.extract(pkg._resolved || pkg._requested || npa.resolve(pkg.name, pkg.version), target, opts).then(() => { log.silly('addBundled', 'read tarball') readPackageTree(target, (err, tree) => { diff --git a/lib/install/action/extract.js b/lib/install/action/extract.js index e8d7a6c4f6d1f..0e84e157670cf 100644 --- a/lib/install/action/extract.js +++ b/lib/install/action/extract.js @@ -2,6 +2,7 @@ const BB = require('bluebird') +const figgyPudding = require('figgy-pudding') const stat = BB.promisify(require('graceful-fs').stat) const gentlyRm = BB.promisify(require('../../utils/gently-rm.js')) const mkdirp = BB.promisify(require('mkdirp')) @@ -9,8 +10,8 @@ const moduleStagingPath = require('../module-staging-path.js') const move = require('../../utils/move.js') const npa = require('npm-package-arg') const npm = require('../../npm.js') +const npmConfig = require('../../config/figgy-config.js') const packageId = require('../../utils/package-id.js') -let pacoteOpts const path = require('path') const localWorker = require('./extract-worker.js') const workerFarm = require('worker-farm') @@ -19,19 +20,12 @@ const isRegistry = require('../../utils/is-registry.js') const WORKER_PATH = require.resolve('./extract-worker.js') let workers -// NOTE: temporarily disabled on non-OSX due to ongoing issues: -// -// * Seems to make Windows antivirus issues much more common -// * Messes with Docker (I think) -// -// There are other issues that should be fixed that affect OSX too: -// -// * Logging is messed up right now because pacote does its own thing -// * Global deduplication in pacote breaks due to multiple procs -// -// As these get fixed, we can start experimenting with re-enabling it -// at least on some platforms. -const ENABLE_WORKERS = process.platform === 'darwin' +const ExtractOpts = figgyPudding({ + log: {} +}, { other () { return true } }) + +// Disabled for now. Re-enable someday. Just not today. +const ENABLE_WORKERS = false extract.init = () => { if (ENABLE_WORKERS) { @@ -53,10 +47,7 @@ module.exports = extract function extract (staging, pkg, log) { log.silly('extract', packageId(pkg)) const extractTo = moduleStagingPath(staging, pkg) - if (!pacoteOpts) { - pacoteOpts = require('../../config/pacote') - } - const opts = pacoteOpts({ + let opts = ExtractOpts(npmConfig()).concat({ integrity: pkg.package._integrity, resolved: pkg.package._resolved }) @@ -72,9 +63,18 @@ function extract (staging, pkg, log) { args[0] = spec.raw if (ENABLE_WORKERS && (isRegistry(spec) || spec.type === 'remote')) { // We can't serialize these options - opts.loglevel = opts.log.level - opts.log = null - opts.dirPacker = null + opts = opts.concat({ + loglevel: opts.log.level, + log: null, + dirPacker: null, + Promise: null, + _events: null, + _eventsCount: null, + list: null, + sources: null, + _maxListeners: null, + root: null + }) // workers will run things in parallel! launcher = workers try { diff --git a/lib/install/action/fetch.js b/lib/install/action/fetch.js index 5ad34e29dd27e..346194e51607e 100644 --- a/lib/install/action/fetch.js +++ b/lib/install/action/fetch.js @@ -3,14 +3,14 @@ const BB = require('bluebird') const finished = BB.promisify(require('mississippi').finished) +const npmConfig = require('../../config/figgy-config.js') const packageId = require('../../utils/package-id.js') const pacote = require('pacote') -const pacoteOpts = require('../../config/pacote') module.exports = fetch function fetch (staging, pkg, log, next) { log.silly('fetch', packageId(pkg)) - const opts = pacoteOpts({integrity: pkg.package._integrity}) + const opts = npmConfig({integrity: pkg.package._integrity}) return finished(pacote.tarball.stream(pkg.package._requested, opts)) .then(() => next(), next) } diff --git a/lib/pack.js b/lib/pack.js index 3b3f5b7bbc700..78e5bfd174d7b 100644 --- a/lib/pack.js +++ b/lib/pack.js @@ -18,9 +18,9 @@ const lifecycle = BB.promisify(require('./utils/lifecycle')) const log = require('npmlog') const move = require('move-concurrently') const npm = require('./npm') +const npmConfig = require('./config/figgy-config.js') const output = require('./utils/output') const pacote = require('pacote') -const pacoteOpts = require('./config/pacote') const path = require('path') const PassThrough = require('stream').PassThrough const pathIsInside = require('path-is-inside') @@ -88,8 +88,8 @@ function pack_ (pkg, dir) { } function packFromPackage (arg, target, filename) { - const opts = pacoteOpts() - return pacote.tarball.toFile(arg, target, pacoteOpts()) + const opts = npmConfig() + return pacote.tarball.toFile(arg, target, opts) .then(() => cacache.tmp.withTmp(npm.tmp, {tmpPrefix: 'unpacking'}, (tmp) => { const tmpTarget = path.join(tmp, filename) return pacote.extract(arg, tmpTarget, opts) diff --git a/lib/publish.js b/lib/publish.js index 7d93615a2c167..e81fc1a057454 100644 --- a/lib/publish.js +++ b/lib/publish.js @@ -14,7 +14,6 @@ const output = require('./utils/output.js') const otplease = require('./utils/otplease.js') const pack = require('./pack') const { tarball, extract } = require('libnpm') -const pacoteOpts = require('./config/pacote') const path = require('path') const readFileAsync = BB.promisify(require('graceful-fs').readFile) const readJson = BB.promisify(require('read-package-json')) @@ -123,9 +122,8 @@ function publishFromPackage (arg, opts) { return cacache.tmp.withTmp(opts.tmp, {tmpPrefix: 'fromPackage'}, tmp => { const extracted = path.join(tmp, 'package') const target = path.join(tmp, 'package.json') - const pacOpts = pacoteOpts() - return tarball.toFile(arg, target, pacOpts) - .then(() => extract(arg, extracted, pacOpts)) + return tarball.toFile(arg, target, opts) + .then(() => extract(arg, extracted, opts)) .then(() => readJson(path.join(extracted, 'package.json'))) .then((pkg) => { return BB.resolve(pack.getContents(pkg, target))