From 7e5d6fb73ccdf6d3d21a30f72f14e08c78bf0c40 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Sat, 3 Feb 2018 00:36:27 -0800 Subject: [PATCH 1/9] WIP: ability to store a built package in offline mirror --- src/cli/commands/pack.js | 2 +- src/lockfile/index.js | 4 +- src/package-install-scripts.js | 57 +++++++++++++++++++++++- src/package-request.js | 2 + src/package-resolver.js | 3 ++ src/resolvers/registries/npm-resolver.js | 9 ++++ src/types.js | 2 + 7 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/cli/commands/pack.js b/src/cli/commands/pack.js index 96e0c50f60..b8bd61f7fc 100644 --- a/src/cli/commands/pack.js +++ b/src/cli/commands/pack.js @@ -148,7 +148,7 @@ export function hasWrapper(commander: Object, args: Array): boolean { return true; } -export async function run(config: Config, reporter: Reporter, flags: Object, args: Array): Promise { +export async function run(config: Config, reporter: Reporter, flags: {filename?: string}, args?: Array): Promise { const pkg = await config.readRootManifest(); if (!pkg.name) { throw new MessageError(reporter.lang('noName')); diff --git a/src/lockfile/index.js b/src/lockfile/index.js index 1edaff55db..510f4281a2 100644 --- a/src/lockfile/index.js +++ b/src/lockfile/index.js @@ -29,6 +29,7 @@ export type LockManifest = { permissions: ?{[key: string]: boolean}, optionalDependencies: ?Dependencies, dependencies: ?Dependencies, + prebuiltVariants: ?{[key: string]: string}, }; type MinimalLockManifest = { @@ -69,6 +70,7 @@ export function implodeEntry(pattern: string, obj: Object): MinimalLockManifest dependencies: blankObjectUndefined(obj.dependencies), optionalDependencies: blankObjectUndefined(obj.optionalDependencies), permissions: blankObjectUndefined(obj.permissions), + prebuiltVariants: blankObjectUndefined(obj.prebuiltVariants), }; } @@ -184,7 +186,6 @@ export default class Lockfile { } continue; } - const obj = implodeEntry(pattern, { name: pkg.name, version: pkg.version, @@ -195,6 +196,7 @@ export default class Lockfile { peerDependencies: pkg.peerDependencies, optionalDependencies: pkg.optionalDependencies, permissions: ref.permissions, + prebuiltVariants: pkg.prebuiltVariants, }); lockfile[pattern] = obj; diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 7b98d841be..2a65ff41dd 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -3,12 +3,16 @@ import type {Manifest} from './types.js'; import type PackageResolver from './package-resolver.js'; import type {Reporter} from './reporters/index.js'; -import type Config from './config.js'; +import Config from './config.js'; import type {ReporterSetSpinner} from './reporters/types.js'; import executeLifecycleScript from './util/execute-lifecycle-script.js'; +import * as crypto from './util/crypto.js'; import * as fs from './util/fs.js'; +import {pack} from './cli/commands/pack.js';Config +const fs2 = require('fs'); const invariant = require('invariant'); +const path = require('path'); const INSTALL_STAGES = ['preinstall', 'install', 'postinstall']; @@ -137,6 +141,13 @@ export default class PackageInstallScripts { return false; } const ref = pkg._reference; + if (pkg.prebuiltVariants) { + for (let variant in pkg.prebuiltVariants) { + if (pkg._remote && pkg._remote.reference && pkg._remote.reference.includes(variant)) { + return false; + } + } + } invariant(ref, 'Missing package reference'); if (!ref.fresh && !this.force) { // this package hasn't been touched @@ -292,6 +303,50 @@ export default class PackageInstallScripts { } } + // generate built package as prebuilt one for offline mirror + for (const pkg of pkgs) { + if (this.packageCanBeInstalled(pkg)) { + const filename = PackageInstallScripts.getPrebuiltName(pkg); + // TODO maybe generated prebuilt packages should be in a subfolder + const filePath = this.config.getOfflineMirrorPath(filename + '.tgz'); + if (!filePath) { + break; + } + const ref = pkg._reference; + invariant(ref, 'expected reference'); + const loc = this.config.generateHardModulePath(ref); + const pkgConfig = await Config.create( + { + cwd: loc, + }, + this.reporter, + ); + const stream = await pack(pkgConfig, loc); + + const hash = await new Promise((resolve, reject) => { + console.log("building", pkg.name) + const validateStream = new crypto.HashStream(); + stream + .pipe(validateStream) + .pipe(fs2.createWriteStream(filePath)) + .on('error', reject) + .on('close', () => resolve(validateStream.getHash())); + }); + // TODO ! don't save artifacts in .yarn-integrity, it is part of the package now + // TODO ! .yarn-integrity should contain prebuiltPackages array now + pkg.prebuiltVariants = pkg.prebuiltVariants || {}; + pkg.prebuiltVariants[filename] = hash; + } + } set.end(); + + } + + static getPrebuiltName(pkg: Manifest): string { + // TODO support platform variant for linux + // TODO support hash for all subdependencies that have installs scripts + const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); + const suffix = `${process.platform}-${process.arch}-${process.versions.modules || ''}`; + return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; } } diff --git a/src/package-request.js b/src/package-request.js index 2e7bf4ac96..758e15c8bc 100644 --- a/src/package-request.js +++ b/src/package-request.js @@ -56,6 +56,7 @@ export default class PackageRequest { optional: boolean; foundInfo: ?Manifest; + // TODO is it Manifest? getLocked(remoteType: string): ?Object { // always prioritise root lockfile const shrunk = this.lockfile.getLocked(this.pattern); @@ -79,6 +80,7 @@ export default class PackageRequest { }, optionalDependencies: shrunk.optionalDependencies, dependencies: shrunk.dependencies, + prebuiltVariants: shrunk.prebuiltVariants, }; } else { return null; diff --git a/src/package-resolver.js b/src/package-resolver.js index 65fdccfb26..aed94b57b8 100644 --- a/src/package-resolver.js +++ b/src/package-resolver.js @@ -105,6 +105,7 @@ export default class PackageResolver { newPkg._remote = ref.remote; newPkg.name = oldPkg.name; newPkg.fresh = oldPkg.fresh; + newPkg.prebuiltVariants = oldPkg.prebuiltVariants; // update patterns for (const pattern of ref.patterns) { @@ -118,6 +119,8 @@ export default class PackageResolver { for (const newPkg of newPkgs) { if (newPkg._reference) { for (const pattern of newPkg._reference.patterns) { + const oldPkg = this.patterns[pattern]; + newPkg.prebuiltVariants = oldPkg.prebuiltVariants; this.patterns[pattern] = newPkg; } } diff --git a/src/resolvers/registries/npm-resolver.js b/src/resolvers/registries/npm-resolver.js index 4cd905fb8a..4a0527b8f6 100644 --- a/src/resolvers/registries/npm-resolver.js +++ b/src/resolvers/registries/npm-resolver.js @@ -3,6 +3,7 @@ import type {Manifest} from '../../types.js'; import type Config from '../../config.js'; import type PackageRequest from '../../package-request.js'; +import PackageInstallScripts from '../../package-install-scripts.js'; import {MessageError} from '../../errors.js'; import RegistryResolver from './registry-resolver.js'; import NpmRegistry, {SCOPE_SEPARATOR} from '../../registries/npm-registry.js'; @@ -180,6 +181,14 @@ export default class NpmResolver extends RegistryResolver { // lockfile const shrunk = this.request.getLocked('tarball'); if (shrunk) { + if (shrunk.prebuiltVariants && shrunk._remote) { + const prebuiltName = PackageInstallScripts.getPrebuiltName(shrunk); + if (shrunk.prebuiltVariants[prebuiltName]) { + const filename = this.config.getOfflineMirrorPath(prebuiltName + '.tgz'); + shrunk._remote.reference = `file:${filename || ''}`; + shrunk._remote.hash = shrunk.prebuiltVariants[prebuiltName]; + } + } return shrunk; } diff --git a/src/types.js b/src/types.js index 7d267d1c29..9906b02c81 100644 --- a/src/types.js +++ b/src/types.js @@ -141,6 +141,8 @@ export type Manifest = { // We need to preserve the flag because we print a list of new packages in // the end of the add command fresh?: boolean, + + prebuiltVariants?: {[filename: string]: string}, }; // From 5d4428e9d748501060613604ff685a136d9605a6 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Wed, 7 Feb 2018 09:11:55 -0800 Subject: [PATCH 2/9] added tests --- __tests__/commands/install/offline-mirror.js | 95 ++++++++++++++++++ .../install-offline-built-artifacts/.yarnrc | 1 + .../mirror-for-offline/dep-a-v1.0.0.tgz | Bin 0 -> 406 bytes .../package.json | 5 + .../install-offline-built-artifacts/yarn.lock | 7 ++ src/cli/commands/pack.js | 7 +- src/config.js | 4 + src/package-install-scripts.js | 72 ++++++------- src/resolvers/registries/npm-resolver.js | 8 +- 9 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 __tests__/commands/install/offline-mirror.js create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts/.yarnrc create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts/mirror-for-offline/dep-a-v1.0.0.tgz create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts/package.json create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts/yarn.lock diff --git a/__tests__/commands/install/offline-mirror.js b/__tests__/commands/install/offline-mirror.js new file mode 100644 index 0000000000..87eb776261 --- /dev/null +++ b/__tests__/commands/install/offline-mirror.js @@ -0,0 +1,95 @@ +/* @flow */ + +import {runInstall} from '../_helpers.js'; +import {Install} from '../../../src/cli/commands/install.js'; +import Lockfile from '../../../src/lockfile'; +import * as fs from '../../../src/util/fs.js'; + +const path = require('path'); + +jasmine.DEFAULT_TIMEOUT_INTERVAL = 150000; + +test.concurrent( + 'install with offline mirror and pack-built-packages setting should run install' + + ' scripts on first call and not run on second while producing the same node_modules', + (): Promise => { + return runInstall({ignoreScripts: true}, 'install-offline-built-artifacts', async (config, reporter) => { + // install scripts were not run + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(false); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); + + // enable packing of built artifacts + config.packBuiltPackages = true; + + // after first run we observe both package and global side effects + let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + + // after second run we observe only package side effects because offline mirror was used + await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); + }); + }, +); + +test.concurrent('install without pack-built-packages should keep running install scripts', (): Promise => { + return runInstall({ignoreScripts: true}, 'install-offline-built-artifacts', async (config, reporter) => { + // install scripts were not run + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(false); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); + + // after first run we observe both package and global side effects + let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + + // after second run we observe both package and global side effects + await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + }); +}); + +test.concurrent('removing prebuilt package .tgz file falls back to running scripts', (): Promise => { + return runInstall({ignoreScripts: true}, 'install-offline-built-artifacts', async (config, reporter) => { + // install scripts were not run + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(false); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); + + // enable packing of built artifacts + config.packBuiltPackages = true; + + // after first run we observe both package and global side effects + let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + + // after second run we observe both package and global side effects + const tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + const packageTgz = tgzFiles.filter(f => f !== 'dep-a-v1.0.0.tgz')[0]; + await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'mirror-for-offline', packageTgz)); + + reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + }); +}); + +// moar tests: +// test platform specific cache +// don't save artifacts in .yarn-integrity: check-integrity should work +// .yarn-integrity should hold which packages are platform specifc and rerun if package changes diff --git a/__tests__/fixtures/install/install-offline-built-artifacts/.yarnrc b/__tests__/fixtures/install/install-offline-built-artifacts/.yarnrc new file mode 100644 index 0000000000..6e653fcdbc --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts/.yarnrc @@ -0,0 +1 @@ +yarn-offline-mirror "./mirror-for-offline" diff --git a/__tests__/fixtures/install/install-offline-built-artifacts/mirror-for-offline/dep-a-v1.0.0.tgz b/__tests__/fixtures/install/install-offline-built-artifacts/mirror-for-offline/dep-a-v1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..54fc2023d603a03d6c5773097d51594881d7c869 GIT binary patch literal 406 zcmV;H0crjpiwFP!000006YW*YPQx$|>^Wbt$|bEt^G+*)xbg#BxtL_r8eF^BPD_RQ z@2p8sMO6fdLWRgEl9SD5cJ1*vOIF^q84i9DaU4&J0^CU0AKQl{OS4Hj&8JxkaS|6r z0)yg5yXw#w%QUs}U7r2P{Sho%{Ua`E9fSz)%<*f`dQ9^Cef@-lo>_9YY?hr0t~MZu&%MR2AFUR8^A|yEZ8|pcWb>Zl+2d%A&y)n`n*zTKq}j2 zwy~K|w{Ho(m!{U5TikUOh7nx{dOL%l+uE>Z26Q8DWZN*2cbor1_Al^mJ&m@}^1?f{ zvY*4N|GSa=ujH|7eB}QmPj=^jnv?&tB*}XJ{|21(e90E*pMziFa?JcQ7h0h~QJ{_UO^0M~{Pe21r!{mH-X_01b@7 AzW@LL literal 0 HcmV?d00001 diff --git a/__tests__/fixtures/install/install-offline-built-artifacts/package.json b/__tests__/fixtures/install/install-offline-built-artifacts/package.json new file mode 100644 index 0000000000..c0d4b7cf22 --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "dep-a": "1.0.0" + } +} diff --git a/__tests__/fixtures/install/install-offline-built-artifacts/yarn.lock b/__tests__/fixtures/install/install-offline-built-artifacts/yarn.lock new file mode 100644 index 0000000000..b9f9beae7b --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts/yarn.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +dep-a@1.0.0: + version "1.0.0" + resolved "./mirror-for-offline/dep-a-v1.0.0.tgz#4ede2f5f2b706cfda23f16adba7e360d4312ec6d" diff --git a/src/cli/commands/pack.js b/src/cli/commands/pack.js index b8bd61f7fc..8f96b5fe97 100644 --- a/src/cli/commands/pack.js +++ b/src/cli/commands/pack.js @@ -148,7 +148,12 @@ export function hasWrapper(commander: Object, args: Array): boolean { return true; } -export async function run(config: Config, reporter: Reporter, flags: {filename?: string}, args?: Array): Promise { +export async function run( + config: Config, + reporter: Reporter, + flags: {filename?: string}, + args?: Array, +): Promise { const pkg = await config.readRootManifest(); if (!pkg.name) { throw new MessageError(reporter.lang('noName')); diff --git a/src/config.js b/src/config.js index 7c869ca199..24f01d8196 100644 --- a/src/config.js +++ b/src/config.js @@ -106,6 +106,9 @@ export default class Config { binLinks: boolean; updateChecksums: boolean; + // cache packages in offline mirror folder as new .tgz files + packBuiltPackages: boolean; + // linkedModules: Array; @@ -328,6 +331,7 @@ export default class Config { this.enableMetaFolder = Boolean(this.getOption('enable-meta-folder')); this.enableLockfileVersions = Boolean(this.getOption('yarn-enable-lockfile-versions')); this.linkFileDependencies = Boolean(this.getOption('yarn-link-file-dependencies')); + this.packBuiltPackages = Boolean(this.getOption('pack-built-packages')); //init & create cacheFolder, tempFolder this.cacheFolder = path.join(this._cacheRootFolder, 'v' + String(constants.CACHE_VERSION)); diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 2a65ff41dd..a6bc373280 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -8,11 +8,10 @@ import type {ReporterSetSpinner} from './reporters/types.js'; import executeLifecycleScript from './util/execute-lifecycle-script.js'; import * as crypto from './util/crypto.js'; import * as fs from './util/fs.js'; -import {pack} from './cli/commands/pack.js';Config +import {pack} from './cli/commands/pack.js'; const fs2 = require('fs'); const invariant = require('invariant'); -const path = require('path'); const INSTALL_STAGES = ['preinstall', 'install', 'postinstall']; @@ -141,8 +140,8 @@ export default class PackageInstallScripts { return false; } const ref = pkg._reference; - if (pkg.prebuiltVariants) { - for (let variant in pkg.prebuiltVariants) { + if (this.config.packBuiltPackages && pkg.prebuiltVariants) { + for (const variant in pkg.prebuiltVariants) { if (pkg._remote && pkg._remote.reference && pkg._remote.reference.includes(variant)) { return false; } @@ -304,42 +303,43 @@ export default class PackageInstallScripts { } // generate built package as prebuilt one for offline mirror - for (const pkg of pkgs) { - if (this.packageCanBeInstalled(pkg)) { - const filename = PackageInstallScripts.getPrebuiltName(pkg); - // TODO maybe generated prebuilt packages should be in a subfolder - const filePath = this.config.getOfflineMirrorPath(filename + '.tgz'); - if (!filePath) { - break; + if (this.config.packBuiltPackages) { + for (const pkg of pkgs) { + if (this.packageCanBeInstalled(pkg)) { + const filename = PackageInstallScripts.getPrebuiltName(pkg); + // TODO maybe generated prebuilt packages should be in a subfolder + const filePath = this.config.getOfflineMirrorPath(filename + '.tgz'); + if (!filePath) { + break; + } + const ref = pkg._reference; + invariant(ref, 'expected reference'); + const loc = this.config.generateHardModulePath(ref); + const pkgConfig = await Config.create( + { + cwd: loc, + }, + this.reporter, + ); + const stream = await pack(pkgConfig, loc); + + const hash = await new Promise((resolve, reject) => { + const validateStream = new crypto.HashStream(); + stream + .pipe(validateStream) + .pipe(fs2.createWriteStream(filePath)) + .on('error', reject) + .on('close', () => resolve(validateStream.getHash())); + }); + // TODO ! don't save artifacts in .yarn-integrity, it is part of the package now + // TODO ! .yarn-integrity should contain prebuiltPackages array now + pkg.prebuiltVariants = pkg.prebuiltVariants || {}; + pkg.prebuiltVariants[filename] = hash; } - const ref = pkg._reference; - invariant(ref, 'expected reference'); - const loc = this.config.generateHardModulePath(ref); - const pkgConfig = await Config.create( - { - cwd: loc, - }, - this.reporter, - ); - const stream = await pack(pkgConfig, loc); - - const hash = await new Promise((resolve, reject) => { - console.log("building", pkg.name) - const validateStream = new crypto.HashStream(); - stream - .pipe(validateStream) - .pipe(fs2.createWriteStream(filePath)) - .on('error', reject) - .on('close', () => resolve(validateStream.getHash())); - }); - // TODO ! don't save artifacts in .yarn-integrity, it is part of the package now - // TODO ! .yarn-integrity should contain prebuiltPackages array now - pkg.prebuiltVariants = pkg.prebuiltVariants || {}; - pkg.prebuiltVariants[filename] = hash; } } - set.end(); + set.end(); } static getPrebuiltName(pkg: Manifest): string { diff --git a/src/resolvers/registries/npm-resolver.js b/src/resolvers/registries/npm-resolver.js index 4a0527b8f6..6ead71400e 100644 --- a/src/resolvers/registries/npm-resolver.js +++ b/src/resolvers/registries/npm-resolver.js @@ -181,12 +181,14 @@ export default class NpmResolver extends RegistryResolver { // lockfile const shrunk = this.request.getLocked('tarball'); if (shrunk) { - if (shrunk.prebuiltVariants && shrunk._remote) { + if (this.config.packBuiltPackages && shrunk.prebuiltVariants && shrunk._remote) { const prebuiltName = PackageInstallScripts.getPrebuiltName(shrunk); if (shrunk.prebuiltVariants[prebuiltName]) { const filename = this.config.getOfflineMirrorPath(prebuiltName + '.tgz'); - shrunk._remote.reference = `file:${filename || ''}`; - shrunk._remote.hash = shrunk.prebuiltVariants[prebuiltName]; + if (filename && (await fs.exists(filename))) { + shrunk._remote.reference = `file:${filename}`; + shrunk._remote.hash = shrunk.prebuiltVariants[prebuiltName]; + } } } return shrunk; From 61dd95893ec23ab6e220374bbac4917e2c0a4f9d Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Wed, 7 Feb 2018 19:13:44 -0800 Subject: [PATCH 3/9] more tests --- __tests__/commands/install/offline-mirror.js | 58 ++++++++++++++++-- .../.yarnrc | 2 + .../mirror-for-offline/dep-a-v1.0.0.tgz | Bin 0 -> 406 bytes .../package.json | 5 ++ .../yarn.lock | 7 +++ src/package-install-scripts.js | 13 +--- src/resolvers/registries/npm-resolver.js | 4 +- src/util/package-name-utils.js | 10 +++ 8 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/mirror-for-offline/dep-a-v1.0.0.tgz create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/package.json create mode 100644 __tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/yarn.lock create mode 100644 src/util/package-name-utils.js diff --git a/__tests__/commands/install/offline-mirror.js b/__tests__/commands/install/offline-mirror.js index 87eb776261..5d304e4d4e 100644 --- a/__tests__/commands/install/offline-mirror.js +++ b/__tests__/commands/install/offline-mirror.js @@ -7,6 +7,16 @@ import * as fs from '../../../src/util/fs.js'; const path = require('path'); +jest.mock('../../../src/util/package-name-utils'); +const nameUtils = jest.requireMock('../../../src/util/package-name-utils'); +beforeEach(() => { + // doing one time mock is tricky, + // found this workaround https://github.com/facebook/jest/issues/2649#issuecomment-360467278 + nameUtils.getPlatformSpecificPackageFilename.mockImplementation( + jest.requireActual('../../../src/util/package-name-utils').getPlatformSpecificPackageFilename, + ); +}); + jasmine.DEFAULT_TIMEOUT_INTERVAL = 150000; test.concurrent( @@ -89,7 +99,47 @@ test.concurrent('removing prebuilt package .tgz file falls back to running scrip }); }); -// moar tests: -// test platform specific cache -// don't save artifacts in .yarn-integrity: check-integrity should work -// .yarn-integrity should hold which packages are platform specifc and rerun if package changes +// This test is not run concurrently because we mock some internal module +test('switching platform for installed node_modules should trigger rebuild / using another prebuilt tgz', (): Promise< + void, +> => { + // TODO force not needed lockfile needs to save when prebuilt components get added + return runInstall({force: true}, 'install-offline-built-artifacts-multiple-platforms', async (config, reporter) => { + let tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + expect(tgzFiles.length).toBe(2); + + // running install with platform 2 (artifacts get rewritten and install scripts rerun) + await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + nameUtils.getPlatformSpecificPackageFilename.mockImplementation(pkg => { + const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); + const suffix = `${process.platform}-${process.arch}-22`; + return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; + }); + + // TODO force not needed invalidate .yarn-integrity based on platform + let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + + tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); + expect(tgzFiles.length).toBe(3); + + // runinng install with platform 1 again (no global side effects) + await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); + await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + nameUtils.getPlatformSpecificPackageFilename.mockImplementation( + jest.requireActual('../../../src/util/package-name-utils').getPlatformSpecificPackageFilename, + ); + + // TODO force not needed invalidate .yarn-integrity based on platform + reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + + tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); + expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); + expect(tgzFiles.length).toBe(3); + }); +}); diff --git a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc new file mode 100644 index 0000000000..c0a55b8e0e --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc @@ -0,0 +1,2 @@ +yarn-offline-mirror "./mirror-for-offline" +pack-built-packages "true" diff --git a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/mirror-for-offline/dep-a-v1.0.0.tgz b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/mirror-for-offline/dep-a-v1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..54fc2023d603a03d6c5773097d51594881d7c869 GIT binary patch literal 406 zcmV;H0crjpiwFP!000006YW*YPQx$|>^Wbt$|bEt^G+*)xbg#BxtL_r8eF^BPD_RQ z@2p8sMO6fdLWRgEl9SD5cJ1*vOIF^q84i9DaU4&J0^CU0AKQl{OS4Hj&8JxkaS|6r z0)yg5yXw#w%QUs}U7r2P{Sho%{Ua`E9fSz)%<*f`dQ9^Cef@-lo>_9YY?hr0t~MZu&%MR2AFUR8^A|yEZ8|pcWb>Zl+2d%A&y)n`n*zTKq}j2 zwy~K|w{Ho(m!{U5TikUOh7nx{dOL%l+uE>Z26Q8DWZN*2cbor1_Al^mJ&m@}^1?f{ zvY*4N|GSa=ujH|7eB}QmPj=^jnv?&tB*}XJ{|21(e90E*pMziFa?JcQ7h0h~QJ{_UO^0M~{Pe21r!{mH-X_01b@7 AzW@LL literal 0 HcmV?d00001 diff --git a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/package.json b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/package.json new file mode 100644 index 0000000000..c0d4b7cf22 --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "dep-a": "1.0.0" + } +} diff --git a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/yarn.lock b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/yarn.lock new file mode 100644 index 0000000000..b9f9beae7b --- /dev/null +++ b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/yarn.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +dep-a@1.0.0: + version "1.0.0" + resolved "./mirror-for-offline/dep-a-v1.0.0.tgz#4ede2f5f2b706cfda23f16adba7e360d4312ec6d" diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index a6bc373280..4cc06395a8 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -8,6 +8,7 @@ import type {ReporterSetSpinner} from './reporters/types.js'; import executeLifecycleScript from './util/execute-lifecycle-script.js'; import * as crypto from './util/crypto.js'; import * as fs from './util/fs.js'; +import {getPlatformSpecificPackageFilename} from './util/package-name-utils.js'; import {pack} from './cli/commands/pack.js'; const fs2 = require('fs'); @@ -306,7 +307,7 @@ export default class PackageInstallScripts { if (this.config.packBuiltPackages) { for (const pkg of pkgs) { if (this.packageCanBeInstalled(pkg)) { - const filename = PackageInstallScripts.getPrebuiltName(pkg); + const filename = getPlatformSpecificPackageFilename(pkg); // TODO maybe generated prebuilt packages should be in a subfolder const filePath = this.config.getOfflineMirrorPath(filename + '.tgz'); if (!filePath) { @@ -331,8 +332,6 @@ export default class PackageInstallScripts { .on('error', reject) .on('close', () => resolve(validateStream.getHash())); }); - // TODO ! don't save artifacts in .yarn-integrity, it is part of the package now - // TODO ! .yarn-integrity should contain prebuiltPackages array now pkg.prebuiltVariants = pkg.prebuiltVariants || {}; pkg.prebuiltVariants[filename] = hash; } @@ -341,12 +340,4 @@ export default class PackageInstallScripts { set.end(); } - - static getPrebuiltName(pkg: Manifest): string { - // TODO support platform variant for linux - // TODO support hash for all subdependencies that have installs scripts - const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); - const suffix = `${process.platform}-${process.arch}-${process.versions.modules || ''}`; - return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; - } } diff --git a/src/resolvers/registries/npm-resolver.js b/src/resolvers/registries/npm-resolver.js index 6ead71400e..4916706c7b 100644 --- a/src/resolvers/registries/npm-resolver.js +++ b/src/resolvers/registries/npm-resolver.js @@ -3,13 +3,13 @@ import type {Manifest} from '../../types.js'; import type Config from '../../config.js'; import type PackageRequest from '../../package-request.js'; -import PackageInstallScripts from '../../package-install-scripts.js'; import {MessageError} from '../../errors.js'; import RegistryResolver from './registry-resolver.js'; import NpmRegistry, {SCOPE_SEPARATOR} from '../../registries/npm-registry.js'; import map from '../../util/map.js'; import * as fs from '../../util/fs.js'; import {YARN_REGISTRY} from '../../constants.js'; +import {getPlatformSpecificPackageFilename} from '../../util/package-name-utils.js'; const inquirer = require('inquirer'); const tty = require('tty'); @@ -182,7 +182,7 @@ export default class NpmResolver extends RegistryResolver { const shrunk = this.request.getLocked('tarball'); if (shrunk) { if (this.config.packBuiltPackages && shrunk.prebuiltVariants && shrunk._remote) { - const prebuiltName = PackageInstallScripts.getPrebuiltName(shrunk); + const prebuiltName = getPlatformSpecificPackageFilename(shrunk); if (shrunk.prebuiltVariants[prebuiltName]) { const filename = this.config.getOfflineMirrorPath(prebuiltName + '.tgz'); if (filename && (await fs.exists(filename))) { diff --git a/src/util/package-name-utils.js b/src/util/package-name-utils.js new file mode 100644 index 0000000000..dcc0a7dac0 --- /dev/null +++ b/src/util/package-name-utils.js @@ -0,0 +1,10 @@ +/* @flow */ +import type {Manifest} from '../types.js'; + +export function getPlatformSpecificPackageFilename(pkg: Manifest): string { + // TODO support platform variant for linux + // TODO support hash for all subdependencies that have installs scripts + const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); + const suffix = `${process.platform}-${process.arch}-${process.versions.modules || ''}`; + return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; +} From 84361604c637ccfed681050cc3c68901d4ef06f1 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Wed, 7 Feb 2018 22:37:27 -0800 Subject: [PATCH 4/9] flow fixes and tests --- __tests__/commands/install/offline-mirror.js | 18 +++++++++--------- src/package-install-scripts.js | 15 ++++++++++----- src/package-request.js | 10 +++++----- src/resolvers/registries/npm-resolver.js | 10 ++++++---- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/__tests__/commands/install/offline-mirror.js b/__tests__/commands/install/offline-mirror.js index 5d304e4d4e..c7fe014af4 100644 --- a/__tests__/commands/install/offline-mirror.js +++ b/__tests__/commands/install/offline-mirror.js @@ -10,7 +10,7 @@ const path = require('path'); jest.mock('../../../src/util/package-name-utils'); const nameUtils = jest.requireMock('../../../src/util/package-name-utils'); beforeEach(() => { - // doing one time mock is tricky, + // doing one time mock for one test is tricky, // found this workaround https://github.com/facebook/jest/issues/2649#issuecomment-360467278 nameUtils.getPlatformSpecificPackageFilename.mockImplementation( jest.requireActual('../../../src/util/package-name-utils').getPlatformSpecificPackageFilename, @@ -86,11 +86,11 @@ test.concurrent('removing prebuilt package .tgz file falls back to running scrip expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); // after second run we observe both package and global side effects - const tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + const tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); const packageTgz = tgzFiles.filter(f => f !== 'dep-a-v1.0.0.tgz')[0]; await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); await fs.unlink(path.join(config.cwd, 'module-a-build.log')); - await fs.unlink(path.join(config.cwd, 'mirror-for-offline', packageTgz)); + await fs.unlink(path.join(config.cwd, 'mirror-for-offline', 'prebuilt', packageTgz)); reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); await reinstall.init(); @@ -105,8 +105,8 @@ test('switching platform for installed node_modules should trigger rebuild / usi > => { // TODO force not needed lockfile needs to save when prebuilt components get added return runInstall({force: true}, 'install-offline-built-artifacts-multiple-platforms', async (config, reporter) => { - let tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); - expect(tgzFiles.length).toBe(2); + let tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); + expect(tgzFiles.length).toBe(1); // running install with platform 2 (artifacts get rewritten and install scripts rerun) await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); @@ -121,10 +121,10 @@ test('switching platform for installed node_modules should trigger rebuild / usi let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); await reinstall.init(); - tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); - expect(tgzFiles.length).toBe(3); + expect(tgzFiles.length).toBe(2); // runinng install with platform 1 again (no global side effects) await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); @@ -137,9 +137,9 @@ test('switching platform for installed node_modules should trigger rebuild / usi reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); await reinstall.init(); - tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline')); + tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(false); - expect(tgzFiles.length).toBe(3); + expect(tgzFiles.length).toBe(2); }); }); diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 4cc06395a8..7825215510 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -13,6 +13,7 @@ import {pack} from './cli/commands/pack.js'; const fs2 = require('fs'); const invariant = require('invariant'); +const path = require('path'); const INSTALL_STAGES = ['preinstall', 'install', 'postinstall']; @@ -307,12 +308,16 @@ export default class PackageInstallScripts { if (this.config.packBuiltPackages) { for (const pkg of pkgs) { if (this.packageCanBeInstalled(pkg)) { - const filename = getPlatformSpecificPackageFilename(pkg); - // TODO maybe generated prebuilt packages should be in a subfolder - const filePath = this.config.getOfflineMirrorPath(filename + '.tgz'); - if (!filePath) { + const offlineMirrorPath = this.config.getOfflineMirrorPath(); + if (!offlineMirrorPath) { break; } + let prebuiltPath = path.join(offlineMirrorPath, 'prebuilt'); + if (!await fs.exists(prebuiltPath)) { + await fs.mkdirp(prebuiltPath); + } + const filename = getPlatformSpecificPackageFilename(pkg); + prebuiltPath = path.join(prebuiltPath, filename + '.tgz'); const ref = pkg._reference; invariant(ref, 'expected reference'); const loc = this.config.generateHardModulePath(ref); @@ -328,7 +333,7 @@ export default class PackageInstallScripts { const validateStream = new crypto.HashStream(); stream .pipe(validateStream) - .pipe(fs2.createWriteStream(filePath)) + .pipe(fs2.createWriteStream(prebuiltPath)) .on('error', reject) .on('close', () => resolve(validateStream.getHash())); }); diff --git a/src/package-request.js b/src/package-request.js index 758e15c8bc..9f336c4f6a 100644 --- a/src/package-request.js +++ b/src/package-request.js @@ -1,6 +1,7 @@ /* @flow */ import type {Dependency, DependencyRequestPattern, Manifest} from './types.js'; +import type {FetcherNames} from './fetchers/index.js'; import type PackageResolver from './package-resolver.js'; import type {Reporter} from './reporters/index.js'; import type Config from './config.js'; @@ -56,8 +57,7 @@ export default class PackageRequest { optional: boolean; foundInfo: ?Manifest; - // TODO is it Manifest? - getLocked(remoteType: string): ?Object { + getLocked(remoteType: FetcherNames): ?Manifest { // always prioritise root lockfile const shrunk = this.lockfile.getLocked(this.pattern); @@ -78,9 +78,9 @@ export default class PackageRequest { hash: resolvedParts.hash, registry: shrunk.registry, }, - optionalDependencies: shrunk.optionalDependencies, - dependencies: shrunk.dependencies, - prebuiltVariants: shrunk.prebuiltVariants, + optionalDependencies: shrunk.optionalDependencies || {}, + dependencies: shrunk.dependencies || {}, + prebuiltVariants: shrunk.prebuiltVariants || {}, }; } else { return null; diff --git a/src/resolvers/registries/npm-resolver.js b/src/resolvers/registries/npm-resolver.js index 4916706c7b..512df56f02 100644 --- a/src/resolvers/registries/npm-resolver.js +++ b/src/resolvers/registries/npm-resolver.js @@ -182,12 +182,14 @@ export default class NpmResolver extends RegistryResolver { const shrunk = this.request.getLocked('tarball'); if (shrunk) { if (this.config.packBuiltPackages && shrunk.prebuiltVariants && shrunk._remote) { + const prebuiltVariants = shrunk.prebuiltVariants; const prebuiltName = getPlatformSpecificPackageFilename(shrunk); - if (shrunk.prebuiltVariants[prebuiltName]) { - const filename = this.config.getOfflineMirrorPath(prebuiltName + '.tgz'); - if (filename && (await fs.exists(filename))) { + const offlineMirrorPath = this.config.getOfflineMirrorPath(); + if (prebuiltVariants[prebuiltName] && offlineMirrorPath) { + const filename = path.join(offlineMirrorPath, 'prebuilt', prebuiltName + '.tgz'); + if (shrunk._remote && (await fs.exists(filename))) { shrunk._remote.reference = `file:${filename}`; - shrunk._remote.hash = shrunk.prebuiltVariants[prebuiltName]; + shrunk._remote.hash = prebuiltVariants[prebuiltName]; } } } From 0af2a4def92563e11c6021832821843f5a1c9281 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Wed, 7 Feb 2018 22:51:32 -0800 Subject: [PATCH 5/9] lockfile updates --- __tests__/commands/install/offline-mirror.js | 3 +-- package.json | 1 + src/cli/commands/install.js | 7 ++++++- yarn.lock | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/__tests__/commands/install/offline-mirror.js b/__tests__/commands/install/offline-mirror.js index c7fe014af4..4e1c111b7b 100644 --- a/__tests__/commands/install/offline-mirror.js +++ b/__tests__/commands/install/offline-mirror.js @@ -103,8 +103,7 @@ test.concurrent('removing prebuilt package .tgz file falls back to running scrip test('switching platform for installed node_modules should trigger rebuild / using another prebuilt tgz', (): Promise< void, > => { - // TODO force not needed lockfile needs to save when prebuilt components get added - return runInstall({force: true}, 'install-offline-built-artifacts-multiple-platforms', async (config, reporter) => { + return runInstall({}, 'install-offline-built-artifacts-multiple-platforms', async (config, reporter) => { let tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); expect(tgzFiles.length).toBe(1); diff --git a/package.json b/package.json index e3bf0303de..908ce8cc0f 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "commander": "^2.9.0", "death": "^1.0.0", "debug": "^2.2.0", + "deepequal": "^0.0.1", "detect-indent": "^5.0.0", "dnscache": "^1.0.1", "glob": "^7.1.1", diff --git a/src/cli/commands/install.js b/src/cli/commands/install.js index f24d57676e..737ca16a5d 100644 --- a/src/cli/commands/install.js +++ b/src/cli/commands/install.js @@ -30,6 +30,7 @@ import WorkspaceLayout from '../../workspace-layout.js'; import ResolutionMap from '../../resolution-map.js'; import guessName from '../../util/guess-name'; +const deepEqual = require('deepequal'); const emoji = require('node-emoji'); const invariant = require('invariant'); const path = require('path'); @@ -809,7 +810,11 @@ export class Install { }); const resolverPatternsAreSameAsInLockfile = Object.keys(lockfileBasedOnResolver).every(pattern => { const manifest = this.lockfile.getLocked(pattern); - return manifest && manifest.resolved === lockfileBasedOnResolver[pattern].resolved; + return ( + manifest && + manifest.resolved === lockfileBasedOnResolver[pattern].resolved && + deepEqual(manifest.prebuiltVariants, lockfileBasedOnResolver[pattern].prebuiltVariants) + ); }); // remove command is followed by install with force, lockfile will be rewritten in any case then diff --git a/yarn.lock b/yarn.lock index fa38ff22eb..890c811fad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1688,6 +1688,13 @@ deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" +deepequal@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/deepequal/-/deepequal-0.0.1.tgz#76780fde807e837140819afc15888504fb6cf875" + dependencies: + fast-apply "*" + is-args "*" + default-require-extensions@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/default-require-extensions/-/default-require-extensions-1.0.0.tgz#f37ea15d3e13ffd9b437d33e1a75b5fb97874cb8" @@ -2172,6 +2179,10 @@ fancy-log@^1.1.0: chalk "^1.1.1" time-stamp "^1.0.0" +fast-apply@*: + version "0.0.3" + resolved "https://registry.yarnpkg.com/fast-apply/-/fast-apply-0.0.3.tgz#7791bb3f7f76b7064c0b73bc3e75bfad8553c861" + fast-deep-equal@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" @@ -2997,6 +3008,10 @@ is-absolute@^0.2.3: is-relative "^0.2.1" is-windows "^0.2.0" +is-args@*: + version "0.0.1" + resolved "https://registry.yarnpkg.com/is-args/-/is-args-0.0.1.tgz#8ca6e3ca557c3b36b3c62aa16a59539185380b16" + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" From 1c92e61b2abe1d46f30c1ebf7297c88114cecd53 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Thu, 8 Feb 2018 17:15:44 -0800 Subject: [PATCH 6/9] fixed more tests --- __tests__/commands/install/integration.js | 166 ----------------- __tests__/commands/install/offline-mirror.js | 184 ++++++++++++++++++- src/integrity-checker.js | 13 +- src/lockfile/index.js | 1 + src/package-install-scripts.js | 33 ++-- src/util/package-name-utils.js | 11 +- 6 files changed, 211 insertions(+), 197 deletions(-) diff --git a/__tests__/commands/install/integration.js b/__tests__/commands/install/integration.js index b606e04a9e..482f75aaeb 100644 --- a/__tests__/commands/install/integration.js +++ b/__tests__/commands/install/integration.js @@ -7,7 +7,6 @@ import {run as cache} from '../../../src/cli/commands/cache.js'; import {run as check} from '../../../src/cli/commands/check.js'; import * as constants from '../../../src/constants.js'; import * as reporters from '../../../src/reporters/index.js'; -import {parse} from '../../../src/lockfile'; import {Install, run as install} from '../../../src/cli/commands/install.js'; import Lockfile from '../../../src/lockfile'; import * as fs from '../../../src/util/fs.js'; @@ -16,7 +15,6 @@ import {getPackageVersion, explodeLockfile, runInstall, runLink, createLockfile, jasmine.DEFAULT_TIMEOUT_INTERVAL = 150000; let request = require('request'); -const semver = require('semver'); const path = require('path'); const stream = require('stream'); @@ -145,22 +143,6 @@ test('reading a lockfile should not optimize it', async () => { }); }); -test('creates the file in the mirror when fetching a git repository', async () => { - await runInstall({}, 'install-git', async (config, reporter): Promise => { - const lockfile = await Lockfile.fromDirectory(config.cwd); - - expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1); - - await fs.unlink(path.join(config.cwd, 'offline-mirror')); - await fs.unlink(path.join(config.cwd, 'node_modules')); - - const firstReinstall = new Install({}, config, reporter, lockfile); - await firstReinstall.init(); - - expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1); - }); -}); - test.concurrent('creates a symlink to a directory when using the link: protocol', async () => { await runInstall({}, 'install-link', async (config): Promise => { const expectPath = path.join(config.cwd, 'node_modules', 'test-absolute'); @@ -570,24 +552,6 @@ test.concurrent('install renamed packages', (): Promise => { }); }); -test.concurrent('install from offline mirror', (): Promise => { - return runInstall({}, 'install-from-offline-mirror', async (config): Promise => { - const allFiles = await fs.walk(config.cwd); - - expect( - allFiles.findIndex((file): boolean => { - return file.relative === path.join('node_modules', 'fake-dependency', 'package.json'); - }), - ).toBeGreaterThanOrEqual(0); - - expect( - allFiles.findIndex((file): boolean => { - return file.relative === path.join('node_modules', '@fakescope', 'fake-dependency', 'package.json'); - }), - ).toBeGreaterThanOrEqual(0); - }); -}); - test.concurrent('install from git cache', (): Promise => { return runInstall({}, 'install-from-git-cache', async (config): Promise => { expect(await getPackageVersion(config, 'dep-a')).toEqual('0.0.1'); @@ -712,26 +676,6 @@ test.concurrent('install should be idempotent', (): Promise => { }); }); -test.concurrent('install should add missing deps to yarn and mirror (PR import scenario)', (): Promise => { - return runInstall({}, 'install-import-pr', async config => { - expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0'); - expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true); - expect(await getPackageVersion(config, 'fake-yarn-dependency')).toEqual('1.0.1'); - - const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline')); - expect(mirror).toHaveLength(3); - expect(mirror[0].relative).toEqual('fake-yarn-dependency-1.0.1.tgz'); - expect(mirror[1].relative.indexOf('mime-db-1.0.')).toEqual(0); - expect(mirror[2].relative).toEqual('mime-types-2.0.0.tgz'); - - const lockFileContent = await fs.readFile(path.join(config.cwd, 'yarn.lock')); - const lockFileLines = explodeLockfile(lockFileContent); - expect(lockFileLines).toHaveLength(11); - expect(lockFileLines[3].indexOf('mime-db@')).toEqual(0); - expect(lockFileLines[6].indexOf('mime-types@2.0.0')).toEqual(0); - }); -}); - test.concurrent('install should update checksums in yarn.lock (--update-checksums)', (): Promise => { const packageRealHash = '5faad9c2c07f60dd76770f71cf025b62a63cfd4e'; const packageCacheName = `npm-abab-1.0.4-${packageRealHash}`; @@ -747,47 +691,6 @@ test.concurrent('install should update checksums in yarn.lock (--update-checksum }); }); -test.concurrent('install should update a dependency to yarn and mirror (PR import scenario 2)', (): Promise => { - // mime-types@2.0.0 is gets updated to mime-types@2.1.11 via - // a change in package.json, - // files in mirror, yarn.lock, package.json and node_modules should reflect that - - return runInstall({}, 'install-import-pr-2', async (config, reporter): Promise => { - expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true); - - expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0'); - - await fs.copy(path.join(config.cwd, 'package.json.after'), path.join(config.cwd, 'package.json'), reporter); - - const reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd)); - await reinstall.init(); - - expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.23.0')).toEqual(true); - - expect(await getPackageVersion(config, 'mime-types')).toEqual('2.1.11'); - - const lockFileWritten = await fs.readFile(path.join(config.cwd, 'yarn.lock')); - const lockFileLines = explodeLockfile(lockFileWritten); - - expect(lockFileLines[0]).toEqual('mime-db@~1.23.0:'); - expect(lockFileLines[2]).toMatch(/resolved "https:\/\/registry\.yarnpkg\.com\/mime-db\/-\/mime-db-/); - - expect(lockFileLines[3]).toEqual('mime-types@2.1.11:'); - expect(lockFileLines[5]).toMatch( - /resolved "https:\/\/registry\.yarnpkg\.com\/mime-types\/-\/mime-types-2\.1\.11\.tgz#[a-f0-9]+"/, - ); - - const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline')); - expect(mirror).toHaveLength(4); - - const newFilesInMirror = mirror.filter((elem): boolean => { - return elem.relative !== 'mime-db-1.0.3.tgz' && elem.relative !== 'mime-types-2.0.0.tgz'; - }); - - expect(newFilesInMirror).toHaveLength(2); - }); -}); - if (process.platform !== 'win32') { // TODO: This seems like a real issue, not just a config issue test.concurrent('install cache symlinks properly', (): Promise => { @@ -805,51 +708,6 @@ if (process.platform !== 'win32') { }); } -test.concurrent('offline mirror can be enabled from parent dir', (): Promise => { - const fixture = { - source: 'offline-mirror-configuration', - cwd: 'enabled-from-parent', - }; - return runInstall({}, fixture, async (config, reporter) => { - const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); - const {object: lockfile} = parse(rawLockfile); - expect(lockfile['mime-types@2.1.14'].resolved).toEqual( - 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', - ); - expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true); - }); -}); - -test.concurrent('offline mirror can be enabled from parent dir, with merging of own .yarnrc', (): Promise => { - const fixture = { - source: 'offline-mirror-configuration', - cwd: 'enabled-from-parent-merge', - }; - return runInstall({}, fixture, async (config, reporter) => { - const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); - const {object: lockfile} = parse(rawLockfile); - expect(lockfile['mime-types@2.1.14'].resolved).toEqual( - 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', - ); - expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true); - }); -}); - -test.concurrent('offline mirror can be disabled locally', (): Promise => { - const fixture = { - source: 'offline-mirror-configuration', - cwd: 'disabled-locally', - }; - return runInstall({}, fixture, async (config, reporter) => { - const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); - const {object: lockfile} = parse(rawLockfile); - expect(lockfile['mime-types@2.1.14'].resolved).toEqual( - 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', - ); - expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(false); - }); -}); - // sync test because we need to get all the requests to confirm their validity test('install a scoped module from authed private registry', (): Promise => { return runInstall({}, 'install-from-authed-private-registry', async config => { @@ -1022,30 +880,6 @@ test.concurrent('should install if symlink source does not exist', async (): Pro await runInstall({}, 'relative-symlinks-work', () => {}); }); -test.concurrent('prunes the offline mirror tarballs after pruning is enabled', (): Promise => { - return runInstall({}, 'prune-offline-mirror', async (config): Promise => { - const mirrorPath = 'mirror-for-offline'; - // Scenario: - // dep-a 1.0.0 was originally installed, and it depends on dep-b 1.0.0, so - // both of these were added to the offline mirror. Then dep-a was upgraded - // to 1.1.0 which doesn't depend on dep-b. After this, pruning was enabled, - // so the next install should remove dep-a-1.0.0.tgz and dep-b-1.0.0.tgz. - expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-a-1.0.0.tgz`))).toEqual(false); - expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-b-1.0.0.tgz`))).toEqual(false); - expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dummy.txt`))).toEqual(true); - }); -}); - -test.concurrent('scoped packages remain in offline mirror after pruning is enabled', (): Promise => { - return runInstall({}, 'prune-offline-mirror-scoped', async (config): Promise => { - const mirrorPath = 'mirror-for-offline'; - // scoped package exists - expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/@fakescope-fake-dependency-1.0.1.tgz`))).toEqual(true); - // unscoped package exists - expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/fake-dependency-1.0.1.tgz`))).toEqual(true); - }); -}); - test.concurrent('bailout should work with --production flag too', (): Promise => { return runInstall({production: true}, 'bailout-prod', async (config, reporter): Promise => { // remove file diff --git a/__tests__/commands/install/offline-mirror.js b/__tests__/commands/install/offline-mirror.js index 4e1c111b7b..7dd4ca6596 100644 --- a/__tests__/commands/install/offline-mirror.js +++ b/__tests__/commands/install/offline-mirror.js @@ -1,17 +1,22 @@ /* @flow */ -import {runInstall} from '../_helpers.js'; +import {runInstall, getPackageVersion, explodeLockfile} from '../_helpers.js'; import {Install} from '../../../src/cli/commands/install.js'; import Lockfile from '../../../src/lockfile'; +import {parse} from '../../../src/lockfile'; import * as fs from '../../../src/util/fs.js'; const path = require('path'); +const semver = require('semver'); jest.mock('../../../src/util/package-name-utils'); const nameUtils = jest.requireMock('../../../src/util/package-name-utils'); beforeEach(() => { // doing one time mock for one test is tricky, // found this workaround https://github.com/facebook/jest/issues/2649#issuecomment-360467278 + nameUtils.getSystemParams.mockImplementation( + jest.requireActual('../../../src/util/package-name-utils').getSystemParams, + ); nameUtils.getPlatformSpecificPackageFilename.mockImplementation( jest.requireActual('../../../src/util/package-name-utils').getPlatformSpecificPackageFilename, ); @@ -110,17 +115,20 @@ test('switching platform for installed node_modules should trigger rebuild / usi // running install with platform 2 (artifacts get rewritten and install scripts rerun) await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + nameUtils.getSystemParams.mockImplementation(pkg => { + return `${process.platform}-${process.arch}-22`; + }); nameUtils.getPlatformSpecificPackageFilename.mockImplementation(pkg => { const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); const suffix = `${process.platform}-${process.arch}-22`; return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; }); - // TODO force not needed invalidate .yarn-integrity based on platform - let reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + let reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd)); await reinstall.init(); tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); + expect(await fs.exists(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log'))).toEqual(true); expect(await fs.exists(path.join(config.cwd, 'module-a-build.log'))).toEqual(true); expect(tgzFiles.length).toBe(2); @@ -128,12 +136,14 @@ test('switching platform for installed node_modules should trigger rebuild / usi // runinng install with platform 1 again (no global side effects) await fs.unlink(path.join(config.cwd, 'node_modules', 'dep-a', 'module-a-build.log')); await fs.unlink(path.join(config.cwd, 'module-a-build.log')); + nameUtils.getSystemParams.mockImplementation( + jest.requireActual('../../../src/util/package-name-utils').getSystemParams, + ); nameUtils.getPlatformSpecificPackageFilename.mockImplementation( jest.requireActual('../../../src/util/package-name-utils').getPlatformSpecificPackageFilename, ); - // TODO force not needed invalidate .yarn-integrity based on platform - reinstall = new Install({force: true}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd)); await reinstall.init(); tgzFiles = await fs.readdir(path.join(config.cwd, 'mirror-for-offline', 'prebuilt')); @@ -142,3 +152,167 @@ test('switching platform for installed node_modules should trigger rebuild / usi expect(tgzFiles.length).toBe(2); }); }); + +test('creates the file in the mirror when fetching a git repository', async () => { + await runInstall({}, 'install-git', async (config, reporter): Promise => { + const lockfile = await Lockfile.fromDirectory(config.cwd); + + expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1); + + await fs.unlink(path.join(config.cwd, 'offline-mirror')); + await fs.unlink(path.join(config.cwd, 'node_modules')); + + const firstReinstall = new Install({}, config, reporter, lockfile); + await firstReinstall.init(); + + expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1); + }); +}); + +test.concurrent('install from offline mirror', (): Promise => { + return runInstall({}, 'install-from-offline-mirror', async (config): Promise => { + const allFiles = await fs.walk(config.cwd); + + expect( + allFiles.findIndex((file): boolean => { + return file.relative === path.join('node_modules', 'fake-dependency', 'package.json'); + }), + ).toBeGreaterThanOrEqual(0); + + expect( + allFiles.findIndex((file): boolean => { + return file.relative === path.join('node_modules', '@fakescope', 'fake-dependency', 'package.json'); + }), + ).toBeGreaterThanOrEqual(0); + }); +}); + +test.concurrent('install should add missing deps to yarn and mirror (PR import scenario)', (): Promise => { + return runInstall({}, 'install-import-pr', async config => { + expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0'); + expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true); + expect(await getPackageVersion(config, 'fake-yarn-dependency')).toEqual('1.0.1'); + + const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline')); + expect(mirror).toHaveLength(3); + expect(mirror[0].relative).toEqual('fake-yarn-dependency-1.0.1.tgz'); + expect(mirror[1].relative.indexOf('mime-db-1.0.')).toEqual(0); + expect(mirror[2].relative).toEqual('mime-types-2.0.0.tgz'); + + const lockFileContent = await fs.readFile(path.join(config.cwd, 'yarn.lock')); + const lockFileLines = explodeLockfile(lockFileContent); + expect(lockFileLines).toHaveLength(11); + expect(lockFileLines[3].indexOf('mime-db@')).toEqual(0); + expect(lockFileLines[6].indexOf('mime-types@2.0.0')).toEqual(0); + }); +}); + +test.concurrent('install should update a dependency to yarn and mirror (PR import scenario 2)', (): Promise => { + // mime-types@2.0.0 is gets updated to mime-types@2.1.11 via + // a change in package.json, + // files in mirror, yarn.lock, package.json and node_modules should reflect that + + return runInstall({}, 'install-import-pr-2', async (config, reporter): Promise => { + expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true); + + expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0'); + + await fs.copy(path.join(config.cwd, 'package.json.after'), path.join(config.cwd, 'package.json'), reporter); + + const reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd)); + await reinstall.init(); + + expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.23.0')).toEqual(true); + + expect(await getPackageVersion(config, 'mime-types')).toEqual('2.1.11'); + + const lockFileWritten = await fs.readFile(path.join(config.cwd, 'yarn.lock')); + const lockFileLines = explodeLockfile(lockFileWritten); + + expect(lockFileLines[0]).toEqual('mime-db@~1.23.0:'); + expect(lockFileLines[2]).toMatch(/resolved "https:\/\/registry\.yarnpkg\.com\/mime-db\/-\/mime-db-/); + + expect(lockFileLines[3]).toEqual('mime-types@2.1.11:'); + expect(lockFileLines[5]).toMatch( + /resolved "https:\/\/registry\.yarnpkg\.com\/mime-types\/-\/mime-types-2\.1\.11\.tgz#[a-f0-9]+"/, + ); + + const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline')); + expect(mirror).toHaveLength(4); + + const newFilesInMirror = mirror.filter((elem): boolean => { + return elem.relative !== 'mime-db-1.0.3.tgz' && elem.relative !== 'mime-types-2.0.0.tgz'; + }); + + expect(newFilesInMirror).toHaveLength(2); + }); +}); + +test.concurrent('offline mirror can be enabled from parent dir', (): Promise => { + const fixture = { + source: 'offline-mirror-configuration', + cwd: 'enabled-from-parent', + }; + return runInstall({}, fixture, async (config, reporter) => { + const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); + const {object: lockfile} = parse(rawLockfile); + expect(lockfile['mime-types@2.1.14'].resolved).toEqual( + 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', + ); + expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true); + }); +}); + +test.concurrent('offline mirror can be enabled from parent dir, with merging of own .yarnrc', (): Promise => { + const fixture = { + source: 'offline-mirror-configuration', + cwd: 'enabled-from-parent-merge', + }; + return runInstall({}, fixture, async (config, reporter) => { + const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); + const {object: lockfile} = parse(rawLockfile); + expect(lockfile['mime-types@2.1.14'].resolved).toEqual( + 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', + ); + expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true); + }); +}); + +test.concurrent('offline mirror can be disabled locally', (): Promise => { + const fixture = { + source: 'offline-mirror-configuration', + cwd: 'disabled-locally', + }; + return runInstall({}, fixture, async (config, reporter) => { + const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock')); + const {object: lockfile} = parse(rawLockfile); + expect(lockfile['mime-types@2.1.14'].resolved).toEqual( + 'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee', + ); + expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(false); + }); +}); + +test.concurrent('prunes the offline mirror tarballs after pruning is enabled', (): Promise => { + return runInstall({}, 'prune-offline-mirror', async (config): Promise => { + const mirrorPath = 'mirror-for-offline'; + // Scenario: + // dep-a 1.0.0 was originally installed, and it depends on dep-b 1.0.0, so + // both of these were added to the offline mirror. Then dep-a was upgraded + // to 1.1.0 which doesn't depend on dep-b. After this, pruning was enabled, + // so the next install should remove dep-a-1.0.0.tgz and dep-b-1.0.0.tgz. + expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-a-1.0.0.tgz`))).toEqual(false); + expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-b-1.0.0.tgz`))).toEqual(false); + expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dummy.txt`))).toEqual(true); + }); +}); + +test.concurrent('scoped packages remain in offline mirror after pruning is enabled', (): Promise => { + return runInstall({}, 'prune-offline-mirror-scoped', async (config): Promise => { + const mirrorPath = 'mirror-for-offline'; + // scoped package exists + expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/@fakescope-fake-dependency-1.0.1.tgz`))).toEqual(true); + // unscoped package exists + expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/fake-dependency-1.0.1.tgz`))).toEqual(true); + }); +}); diff --git a/src/integrity-checker.js b/src/integrity-checker.js index 3dcd33c14f..9f6102b9bb 100644 --- a/src/integrity-checker.js +++ b/src/integrity-checker.js @@ -5,6 +5,7 @@ import type {LockManifest} from './lockfile'; import * as constants from './constants.js'; import * as fs from './util/fs.js'; import {sortAlpha, compareSortedArrays} from './util/misc.js'; +import {getSystemParams} from './util/package-name-utils.js'; import type {InstallArtifacts} from './package-install-scripts.js'; import WorkspaceLayout from './workspace-layout.js'; @@ -19,7 +20,7 @@ export const integrityErrors = { LINKED_MODULES_DONT_MATCH: 'integrityCheckLinkedModulesDontMatch', PATTERNS_DONT_MATCH: 'integrityPatternsDontMatch', MODULES_FOLDERS_MISSING: 'integrityModulesFoldersMissing', - NODE_VERSION_DOESNT_MATCH: 'integrityNodeDoesntMatch', + SYSTEM_PARAMS_DONT_MATCH: 'integritySystemParamsDontMatch', }; type IntegrityError = $Keys; @@ -38,7 +39,7 @@ type IntegrityHashLocation = { }; type IntegrityFile = { - nodeVersion: string, + systemParams: string, flags: Array, modulesFolders: Array, linkedModules: Array, @@ -56,7 +57,7 @@ type IntegrityFlags = { }; const INTEGRITY_FILE_DEFAULTS = () => ({ - nodeVersion: process.version, + systemParams: getSystemParams(), modulesFolders: [], flags: [], linkedModules: [], @@ -298,8 +299,8 @@ export default class InstallationIntegrityChecker { return 'LINKED_MODULES_DONT_MATCH'; } - if (actual.nodeVersion !== expected.nodeVersion) { - return 'NODE_VERSION_DOESNT_MATCH'; + if (actual.systemParams !== expected.systemParams) { + return 'SYSTEM_PARAMS_DONT_MATCH'; } let relevantExpectedFlags = expected.flags.slice(); @@ -393,7 +394,7 @@ export default class InstallationIntegrityChecker { integrityMatches: integrityMatches === 'OK', integrityError: integrityMatches === 'OK' ? undefined : integrityMatches, missingPatterns, - hardRefreshRequired: integrityMatches === 'NODE_VERSION_DOESNT_MATCH', + hardRefreshRequired: integrityMatches === 'SYSTEM_PARAMS_DONT_MATCH', }; } diff --git a/src/lockfile/index.js b/src/lockfile/index.js index 510f4281a2..381a7f6ea9 100644 --- a/src/lockfile/index.js +++ b/src/lockfile/index.js @@ -198,6 +198,7 @@ export default class Lockfile { permissions: ref.permissions, prebuiltVariants: pkg.prebuiltVariants, }); + lockfile[pattern] = obj; if (remoteKey) { diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 7825215510..2624a534c2 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -141,7 +141,6 @@ export default class PackageInstallScripts { if (!cmds.length) { return false; } - const ref = pkg._reference; if (this.config.packBuiltPackages && pkg.prebuiltVariants) { for (const variant in pkg.prebuiltVariants) { if (pkg._remote && pkg._remote.reference && pkg._remote.reference.includes(variant)) { @@ -149,6 +148,7 @@ export default class PackageInstallScripts { } } } + const ref = pkg._reference; invariant(ref, 'Missing package reference'); if (!ref.fresh && !this.force) { // this package hasn't been touched @@ -292,18 +292,6 @@ export default class PackageInstallScripts { await Promise.all(workers); - // cache all build artifacts - for (const pkg of pkgs) { - if (this.packageCanBeInstalled(pkg)) { - const ref = pkg._reference; - invariant(ref, 'expected reference'); - const loc = this.config.generateHardModulePath(ref); - const beforeFiles = beforeFilesMap.get(loc); - invariant(beforeFiles, 'files before installation should always be recorded'); - await this.saveBuildArtifacts(loc, pkg, beforeFiles, set.spinners[0]); - } - } - // generate built package as prebuilt one for offline mirror if (this.config.packBuiltPackages) { for (const pkg of pkgs) { @@ -316,8 +304,8 @@ export default class PackageInstallScripts { if (!await fs.exists(prebuiltPath)) { await fs.mkdirp(prebuiltPath); } - const filename = getPlatformSpecificPackageFilename(pkg); - prebuiltPath = path.join(prebuiltPath, filename + '.tgz'); + const prebuiltFilename = getPlatformSpecificPackageFilename(pkg); + prebuiltPath = path.join(prebuiltPath, prebuiltFilename + '.tgz'); const ref = pkg._reference; invariant(ref, 'expected reference'); const loc = this.config.generateHardModulePath(ref); @@ -338,7 +326,20 @@ export default class PackageInstallScripts { .on('close', () => resolve(validateStream.getHash())); }); pkg.prebuiltVariants = pkg.prebuiltVariants || {}; - pkg.prebuiltVariants[filename] = hash; + pkg.prebuiltVariants[prebuiltFilename] = hash; + // this.artifacts[`${pkg.name}@${pkg.version}`] = [prebuiltFilename]; + } + } + } else { + // cache all build artifacts + for (const pkg of pkgs) { + if (this.packageCanBeInstalled(pkg)) { + const ref = pkg._reference; + invariant(ref, 'expected reference'); + const loc = this.config.generateHardModulePath(ref); + const beforeFiles = beforeFilesMap.get(loc); + invariant(beforeFiles, 'files before installation should always be recorded'); + await this.saveBuildArtifacts(loc, pkg, beforeFiles, set.spinners[0]); } } } diff --git a/src/util/package-name-utils.js b/src/util/package-name-utils.js index dcc0a7dac0..89d33ce1e5 100644 --- a/src/util/package-name-utils.js +++ b/src/util/package-name-utils.js @@ -1,10 +1,13 @@ /* @flow */ -import type {Manifest} from '../types.js'; -export function getPlatformSpecificPackageFilename(pkg: Manifest): string { - // TODO support platform variant for linux +export function getPlatformSpecificPackageFilename(pkg: {name: string, version: string}): string { // TODO support hash for all subdependencies that have installs scripts const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); - const suffix = `${process.platform}-${process.arch}-${process.versions.modules || ''}`; + const suffix = getSystemParams(); return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; } + +export function getSystemParams(): string { + // TODO support platform variant for linux + return `${process.platform}-${process.arch}-${process.versions.modules || ''}`; +} From c6fc6403760ca3e99ed3def78f1ff59e7c3d1af0 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Sun, 18 Feb 2018 17:47:33 -0800 Subject: [PATCH 7/9] feedback --- src/config.js | 2 +- src/package-install-scripts.js | 13 +++++-------- src/util/package-name-utils.js | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/config.js b/src/config.js index 24f01d8196..a37eaee047 100644 --- a/src/config.js +++ b/src/config.js @@ -331,7 +331,7 @@ export default class Config { this.enableMetaFolder = Boolean(this.getOption('enable-meta-folder')); this.enableLockfileVersions = Boolean(this.getOption('yarn-enable-lockfile-versions')); this.linkFileDependencies = Boolean(this.getOption('yarn-link-file-dependencies')); - this.packBuiltPackages = Boolean(this.getOption('pack-built-packages')); + this.packBuiltPackages = Boolean(this.getOption('experimental-pack-script-packages-in-mirror')); //init & create cacheFolder, tempFolder this.cacheFolder = path.join(this._cacheRootFolder, 'v' + String(constants.CACHE_VERSION)); diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 2624a534c2..01bea9cb67 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -293,13 +293,10 @@ export default class PackageInstallScripts { await Promise.all(workers); // generate built package as prebuilt one for offline mirror - if (this.config.packBuiltPackages) { + const offlineMirrorPath = this.config.getOfflineMirrorPath(); + if (this.config.packBuiltPackages && offlineMirrorPath) { for (const pkg of pkgs) { if (this.packageCanBeInstalled(pkg)) { - const offlineMirrorPath = this.config.getOfflineMirrorPath(); - if (!offlineMirrorPath) { - break; - } let prebuiltPath = path.join(offlineMirrorPath, 'prebuilt'); if (!await fs.exists(prebuiltPath)) { await fs.mkdirp(prebuiltPath); @@ -308,14 +305,14 @@ export default class PackageInstallScripts { prebuiltPath = path.join(prebuiltPath, prebuiltFilename + '.tgz'); const ref = pkg._reference; invariant(ref, 'expected reference'); - const loc = this.config.generateHardModulePath(ref); + const builtPackagePath = this.config.generateHardModulePath(ref); const pkgConfig = await Config.create( { - cwd: loc, + cwd: builtPackagePath, }, this.reporter, ); - const stream = await pack(pkgConfig, loc); + const stream = await pack(pkgConfig, builtPackagePath); const hash = await new Promise((resolve, reject) => { const validateStream = new crypto.HashStream(); diff --git a/src/util/package-name-utils.js b/src/util/package-name-utils.js index 89d33ce1e5..29bd9de430 100644 --- a/src/util/package-name-utils.js +++ b/src/util/package-name-utils.js @@ -2,9 +2,9 @@ export function getPlatformSpecificPackageFilename(pkg: {name: string, version: string}): string { // TODO support hash for all subdependencies that have installs scripts - const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); + const normalizeScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name); const suffix = getSystemParams(); - return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`; + return `${normalizeScope(pkg.name)}-v${pkg.version}-${suffix}`; } export function getSystemParams(): string { From b8f8a8133cadf0609250b422e3ef7b481b11e952 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Sun, 18 Feb 2018 17:57:35 -0800 Subject: [PATCH 8/9] fixed test --- .../install-offline-built-artifacts-multiple-platforms/.yarnrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc index c0a55b8e0e..16aa5f3864 100644 --- a/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc +++ b/__tests__/fixtures/install/install-offline-built-artifacts-multiple-platforms/.yarnrc @@ -1,2 +1,2 @@ yarn-offline-mirror "./mirror-for-offline" -pack-built-packages "true" +experimental-pack-script-packages-in-mirror "true" From 11234fd00c970ba873be36967949122891d575ad Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Tue, 20 Feb 2018 22:22:10 -0800 Subject: [PATCH 9/9] feedback --- src/package-install-scripts.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/package-install-scripts.js b/src/package-install-scripts.js index 01bea9cb67..16d4da05a7 100644 --- a/src/package-install-scripts.js +++ b/src/package-install-scripts.js @@ -7,11 +7,11 @@ import Config from './config.js'; import type {ReporterSetSpinner} from './reporters/types.js'; import executeLifecycleScript from './util/execute-lifecycle-script.js'; import * as crypto from './util/crypto.js'; -import * as fs from './util/fs.js'; +import * as fsUtil from './util/fs.js'; import {getPlatformSpecificPackageFilename} from './util/package-name-utils.js'; import {pack} from './cli/commands/pack.js'; -const fs2 = require('fs'); +const fs = require('fs'); const invariant = require('invariant'); const path = require('path'); @@ -68,7 +68,7 @@ export default class PackageInstallScripts { } async walk(loc: string): Promise> { - const files = await fs.walk(loc, null, new Set(this.config.registryFolders)); + const files = await fsUtil.walk(loc, null, new Set(this.config.registryFolders)); const mtimes = new Map(); for (const file of files) { mtimes.set(file.relative, file.mtime); @@ -126,7 +126,7 @@ export default class PackageInstallScripts { // Cleanup node_modules try { - await fs.unlink(loc); + await fsUtil.unlink(loc); } catch (e) { this.reporter.error(this.reporter.lang('optionalModuleCleanupFail', e.message)); } @@ -298,9 +298,7 @@ export default class PackageInstallScripts { for (const pkg of pkgs) { if (this.packageCanBeInstalled(pkg)) { let prebuiltPath = path.join(offlineMirrorPath, 'prebuilt'); - if (!await fs.exists(prebuiltPath)) { - await fs.mkdirp(prebuiltPath); - } + await fsUtil.mkdirp(prebuiltPath); const prebuiltFilename = getPlatformSpecificPackageFilename(pkg); prebuiltPath = path.join(prebuiltPath, prebuiltFilename + '.tgz'); const ref = pkg._reference; @@ -318,13 +316,12 @@ export default class PackageInstallScripts { const validateStream = new crypto.HashStream(); stream .pipe(validateStream) - .pipe(fs2.createWriteStream(prebuiltPath)) + .pipe(fs.createWriteStream(prebuiltPath)) .on('error', reject) .on('close', () => resolve(validateStream.getHash())); }); pkg.prebuiltVariants = pkg.prebuiltVariants || {}; pkg.prebuiltVariants[prebuiltFilename] = hash; - // this.artifacts[`${pkg.name}@${pkg.version}`] = [prebuiltFilename]; } } } else {