From 5e83f0119a5858f15fbcb5342581cd714149f9a3 Mon Sep 17 00:00:00 2001 From: Georgios Kalpakas Date: Wed, 9 Sep 2015 03:27:45 +0300 Subject: [PATCH 1/2] chore(check-node-modules): make check/reinstall node_modules work across platforms The previous implementations (based on shell scripts) threw errors on Windows, because it was not able to `rm -rf` 'node_modules' (due to the 255 character limit in file-paths). This implementation works consistently across platforms and is heavily based on 'https://github.com/angular/angular/blob/3b9c08676a4c921bbfa847802e08566fb601ba7a/tools/npm/check-node-modules.js'. Fixes #11143 Closes #11353 --- .travis.yml | 2 +- Gruntfile.js | 2 +- scripts/npm/check-node-modules.js | 175 ++++++++++++++++++++++++++++ scripts/npm/install-dependencies.sh | 16 --- 4 files changed, 177 insertions(+), 18 deletions(-) create mode 100644 scripts/npm/check-node-modules.js delete mode 100755 scripts/npm/install-dependencies.sh diff --git a/.travis.yml b/.travis.yml index db65c2d1bcef..28623e9c9478 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ install: - npm config set loglevel http - npm install -g npm@2.5 # Instal npm dependecies and ensure that npm cache is not stale - - scripts/npm/install-dependencies.sh + - node scripts/npm/check-node-modules.js --reinstall before_script: - mkdir -p $LOGS_DIR diff --git a/Gruntfile.js b/Gruntfile.js index 53aaf59a9ad6..018f08b16f1c 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -305,7 +305,7 @@ module.exports = function(grunt) { shell: { "npm-install": { - command: path.normalize('scripts/npm/install-dependencies.sh') + command: 'node scripts/npm/check-node-modules.js' }, "promises-aplus-tests": { diff --git a/scripts/npm/check-node-modules.js b/scripts/npm/check-node-modules.js new file mode 100644 index 000000000000..bd93847de995 --- /dev/null +++ b/scripts/npm/check-node-modules.js @@ -0,0 +1,175 @@ +// Implementation based on +// https://github.com/angular/angular/blob/3b9c08676a4c921bbfa847802e08566fb601ba7a/tools/npm/check-node-modules.js +'use strict'; + +// Imports +var fs = require('fs'); +var path = require('path'); + +// Constants +var NPM_SHRINKWRAP_FILE = 'npm-shrinkwrap.json'; +var NPM_SHRINKWRAP_CACHED_FILE = 'node_modules/npm-shrinkwrap.cached.json'; +var FS_OPTS = {encoding: 'utf-8'}; +var PROJECT_ROOT = path.join(__dirname, '../../'); + +// Variables +var progressIndicatorTimer = null; + +// Run +_main(); + +// Functions - Definitions +function _main() { + var reinstallIfStale = process.argv.indexOf('--reinstall') !== -1; + checkNodeModules(reinstallIfStale); +} + +function cacheNpmShrinkwrapFile() { + var absoluteMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_FILE); + var absoluteCachedMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_CACHED_FILE); + + startProgressIndicator(' Caching marker file...'); + copyFile(absoluteMarkerFilePath, absoluteCachedMarkerFilePath, onCopied); + + // Helpers + function onCopied(err) { + stopProgressIndicator(); + if (err) logError(err); + } +} + +function checkNodeModules(reinstallIfStale) { + var nodeModulesOk = compareMarkerFiles(); + + if (nodeModulesOk) { + console.log(':-) npm dependencies are looking good!'); + } else { + console.warn(':-( npm dependencies are stale or in an unknown state!'); + + if (reinstallIfStale) { + purgeModules(); + installModules(); + } + } + + return nodeModulesOk; +} + +function compareMarkerFiles() { + var absoluteMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_FILE); + var absoluteCachedMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_CACHED_FILE); + + if (!fs.existsSync(absoluteCachedMarkerFilePath)) return false; + + var markerContent = fs.readFileSync(absoluteMarkerFilePath, FS_OPTS); + var cachedMarkerContent = fs.readFileSync(absoluteCachedMarkerFilePath, FS_OPTS); + + return markerContent === cachedMarkerContent; +} + +// Implementation based on +// https://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js#answer-21995878 +function copyFile(srcPath, dstPath, callback) { + var callbackCalled = false; + + if (!fs.existsSync(srcPath)) { + done(new Error('Missing source file: ' + srcPath)); + return; + } + + var rs = fs.createReadStream(srcPath); + rs.on('error', done); + + var ws = fs.createWriteStream(dstPath); + ws.on('error', done); + ws.on('finish', done); + + rs.pipe(ws); + + // Helpers + function done(err) { + if (callback && !callbackCalled) { + callbackCalled = true; + callback(err); + } + } +} + +// Custom implementation of `rm -rf` that works consistently across OSes +function deleteDirSync(path) { + if (fs.existsSync(path)) { + fs.readdirSync(path).forEach(deleteDirOrFileSync); + fs.rmdirSync(path); + } + + // Helpers + function deleteDirOrFileSync(subpath) { + var curPath = path + '/' + subpath; + + if (fs.lstatSync(curPath).isDirectory()) { + deleteDirSync(curPath); + } else { + fs.unlinkSync(curPath); + } + } +} + +function installModules() { + startProgressIndicator(' Running `npm install` (this may take a while)...'); + + var exec = require('child_process').exec; + var opts = {cwd: PROJECT_ROOT}; + var proc = exec('npm install', opts, onInstalled); + + // TODO(gkalpak): Decide if we actually want these + // proc.stdout.pipe(process.stdout); + // proc.stderr.pipe(process.stderr); + + // Helpers + function onInstalled(err) { + stopProgressIndicator(); + + if (err) { + logError(err); + return; + } + + cacheNpmShrinkwrapFile(); + } +} + +function purgeModules() { + startProgressIndicator(' Purging \'node_modules\'...'); + + var nodeModulesPath = path.join(PROJECT_ROOT, 'node_modules'); + deleteDirSync(nodeModulesPath); + + stopProgressIndicator(); +} + +function logError(err) { + var separator = new Array(81).join('!'); + + console.error(separator); + console.error('Operation completed with errors:'); + console.error(err); + console.error(separator); +} + +function startProgressIndicator(taskDescription) { + stopProgressIndicator(); + + var stdout = process.stdout; + + stdout.write(taskDescription); + progressIndicatorTimer = setInterval(stdout.write.bind(stdout, '.'), 5000); +} + +function stopProgressIndicator() { + if (progressIndicatorTimer) { + clearInterval(progressIndicatorTimer); + progressIndicatorTimer = null; + + process.stdout.write('\n'); + } +} diff --git a/scripts/npm/install-dependencies.sh b/scripts/npm/install-dependencies.sh deleted file mode 100755 index 1851d4ea6ae4..000000000000 --- a/scripts/npm/install-dependencies.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -SHRINKWRAP_FILE=npm-shrinkwrap.json -SHRINKWRAP_CACHED_FILE=node_modules/npm-shrinkwrap.cached.json - -if diff -q $SHRINKWRAP_FILE $SHRINKWRAP_CACHED_FILE; then - echo 'No shrinkwrap changes detected. npm install will be skipped...'; -else - echo 'Blowing away node_modules and reinstalling npm dependencies...' - rm -rf node_modules - npm install - cp $SHRINKWRAP_FILE $SHRINKWRAP_CACHED_FILE - echo 'npm install successful!' -fi From 0b36dcece54ff2a30e9f1afd9c1937cc34a694fe Mon Sep 17 00:00:00 2001 From: Georgios Kalpakas Date: Tue, 15 Sep 2015 18:15:46 +0300 Subject: [PATCH 2/2] WIP - fixup: use npm's `preinstall`/`postinstall` hooks Changes: 1. Make warning about stale dependencies stand out. 2. Check 'node_modules/' and purge at `preinstall` phase. 3. Copy 'npm-shrinkwrap.json' at `postinstall` phase. 4. Just run `npm install` to install dependencies (e.g. in '.travis.yml'). --- .travis.yml | 2 +- package.json | 4 + scripts/npm/check-node-modules.js | 147 +++++------------------------ scripts/npm/copy-npm-shrinkwrap.js | 60 ++++++++++++ 4 files changed, 88 insertions(+), 125 deletions(-) create mode 100644 scripts/npm/copy-npm-shrinkwrap.js diff --git a/.travis.yml b/.travis.yml index 28623e9c9478..27b81cc5735a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ install: - npm config set loglevel http - npm install -g npm@2.5 # Instal npm dependecies and ensure that npm cache is not stale - - node scripts/npm/check-node-modules.js --reinstall + - npm install before_script: - mkdir -p $LOGS_DIR diff --git a/package.json b/package.json index 2eaffa43e9d9..00556b312722 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,10 @@ "npm": "~2.5" }, "engineStrict": true, + "scripts": { + "preinstall": "node scripts/npm/check-node-modules.js --purge", + "postinstall": "node scripts/npm/copy-npm-shrinkwrap.js" + }, "devDependencies": { "angular-benchpress": "0.x.x", "benchmark": "1.x.x", diff --git a/scripts/npm/check-node-modules.js b/scripts/npm/check-node-modules.js index bd93847de995..a9bc24a9a593 100644 --- a/scripts/npm/check-node-modules.js +++ b/scripts/npm/check-node-modules.js @@ -1,4 +1,4 @@ -// Implementation based on +// Implementation based on: // https://github.com/angular/angular/blob/3b9c08676a4c921bbfa847802e08566fb601ba7a/tools/npm/check-node-modules.js 'use strict'; @@ -7,94 +7,53 @@ var fs = require('fs'); var path = require('path'); // Constants -var NPM_SHRINKWRAP_FILE = 'npm-shrinkwrap.json'; -var NPM_SHRINKWRAP_CACHED_FILE = 'node_modules/npm-shrinkwrap.cached.json'; -var FS_OPTS = {encoding: 'utf-8'}; var PROJECT_ROOT = path.join(__dirname, '../../'); - -// Variables -var progressIndicatorTimer = null; +var NODE_MODULES_DIR = 'node_modules'; +var NPM_SHRINKWRAP_FILE = 'npm-shrinkwrap.json'; +var NPM_SHRINKWRAP_CACHED_FILE = NODE_MODULES_DIR + '/npm-shrinkwrap.cached.json'; // Run _main(); // Functions - Definitions function _main() { - var reinstallIfStale = process.argv.indexOf('--reinstall') !== -1; - checkNodeModules(reinstallIfStale); -} - -function cacheNpmShrinkwrapFile() { - var absoluteMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_FILE); - var absoluteCachedMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_CACHED_FILE); - - startProgressIndicator(' Caching marker file...'); - copyFile(absoluteMarkerFilePath, absoluteCachedMarkerFilePath, onCopied); + var purgeIfStale = process.argv.indexOf('--purge') !== -1; - // Helpers - function onCopied(err) { - stopProgressIndicator(); - if (err) logError(err); - } + process.chdir(PROJECT_ROOT); + checkNodeModules(purgeIfStale); } -function checkNodeModules(reinstallIfStale) { - var nodeModulesOk = compareMarkerFiles(); +function checkNodeModules(purgeIfStale) { + var nodeModulesOk = compareMarkerFiles(NPM_SHRINKWRAP_FILE, NPM_SHRINKWRAP_CACHED_FILE); if (nodeModulesOk) { console.log(':-) npm dependencies are looking good!'); + } else if (purgeIfStale) { + console.log(':-( npm dependencies are stale or in an unknown state!'); + console.log(' Purging \'' + NODE_MODULES_DIR + '\'...'); + deleteDirSync(NODE_MODULES_DIR); } else { - console.warn(':-( npm dependencies are stale or in an unknown state!'); + var separator = new Array(81).join('!'); - if (reinstallIfStale) { - purgeModules(); - installModules(); - } + console.warn(separator); + console.warn(':-( npm dependencies are stale or in an unknown state!'); + console.warn('You can rebuild the dependencies by running `npm install`.'); + console.warn(separator); } return nodeModulesOk; } -function compareMarkerFiles() { - var absoluteMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_FILE); - var absoluteCachedMarkerFilePath = path.join(PROJECT_ROOT, NPM_SHRINKWRAP_CACHED_FILE); - - if (!fs.existsSync(absoluteCachedMarkerFilePath)) return false; +function compareMarkerFiles(markerFilePath, cachedMarkerFilePath) { + if (!fs.existsSync(cachedMarkerFilePath)) return false; - var markerContent = fs.readFileSync(absoluteMarkerFilePath, FS_OPTS); - var cachedMarkerContent = fs.readFileSync(absoluteCachedMarkerFilePath, FS_OPTS); + var opts = {encoding: 'utf-8'}; + var markerContent = fs.readFileSync(markerFilePath, opts); + var cachedMarkerContent = fs.readFileSync(cachedMarkerFilePath, opts); return markerContent === cachedMarkerContent; } -// Implementation based on -// https://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js#answer-21995878 -function copyFile(srcPath, dstPath, callback) { - var callbackCalled = false; - - if (!fs.existsSync(srcPath)) { - done(new Error('Missing source file: ' + srcPath)); - return; - } - - var rs = fs.createReadStream(srcPath); - rs.on('error', done); - - var ws = fs.createWriteStream(dstPath); - ws.on('error', done); - ws.on('finish', done); - - rs.pipe(ws); - - // Helpers - function done(err) { - if (callback && !callbackCalled) { - callbackCalled = true; - callback(err); - } - } -} - // Custom implementation of `rm -rf` that works consistently across OSes function deleteDirSync(path) { if (fs.existsSync(path)) { @@ -113,63 +72,3 @@ function deleteDirSync(path) { } } } - -function installModules() { - startProgressIndicator(' Running `npm install` (this may take a while)...'); - - var exec = require('child_process').exec; - var opts = {cwd: PROJECT_ROOT}; - var proc = exec('npm install', opts, onInstalled); - - // TODO(gkalpak): Decide if we actually want these - // proc.stdout.pipe(process.stdout); - // proc.stderr.pipe(process.stderr); - - // Helpers - function onInstalled(err) { - stopProgressIndicator(); - - if (err) { - logError(err); - return; - } - - cacheNpmShrinkwrapFile(); - } -} - -function purgeModules() { - startProgressIndicator(' Purging \'node_modules\'...'); - - var nodeModulesPath = path.join(PROJECT_ROOT, 'node_modules'); - deleteDirSync(nodeModulesPath); - - stopProgressIndicator(); -} - -function logError(err) { - var separator = new Array(81).join('!'); - - console.error(separator); - console.error('Operation completed with errors:'); - console.error(err); - console.error(separator); -} - -function startProgressIndicator(taskDescription) { - stopProgressIndicator(); - - var stdout = process.stdout; - - stdout.write(taskDescription); - progressIndicatorTimer = setInterval(stdout.write.bind(stdout, '.'), 5000); -} - -function stopProgressIndicator() { - if (progressIndicatorTimer) { - clearInterval(progressIndicatorTimer); - progressIndicatorTimer = null; - - process.stdout.write('\n'); - } -} diff --git a/scripts/npm/copy-npm-shrinkwrap.js b/scripts/npm/copy-npm-shrinkwrap.js new file mode 100644 index 000000000000..78bfe3baaafa --- /dev/null +++ b/scripts/npm/copy-npm-shrinkwrap.js @@ -0,0 +1,60 @@ +'use strict'; + +// Imports +var fs = require('fs'); +var path = require('path'); + +// Constants +var PROJECT_ROOT = path.join(__dirname, '../../'); +var NODE_MODULES_DIR = 'node_modules'; +var NPM_SHRINKWRAP_FILE = 'npm-shrinkwrap.json'; +var NPM_SHRINKWRAP_CACHED_FILE = NODE_MODULES_DIR + '/npm-shrinkwrap.cached.json'; + +// Run +_main(); + +// Functions - Definitions +function _main() { + process.chdir(PROJECT_ROOT); + copyFile(NPM_SHRINKWRAP_FILE, NPM_SHRINKWRAP_CACHED_FILE, onCopied); +} + +// Implementation based on: +// https://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js#answer-21995878 +function copyFile(srcPath, dstPath, callback) { + var callbackCalled = false; + + if (!fs.existsSync(srcPath)) { + done(new Error('Missing source file: ' + srcPath)); + return; + } + + var rs = fs.createReadStream(srcPath); + rs.on('error', done); + + var ws = fs.createWriteStream(dstPath); + ws.on('error', done); + ws.on('finish', done); + + rs.pipe(ws); + + // Helpers + function done(err) { + if (callback && !callbackCalled) { + callbackCalled = true; + callback(err); + } + } +} + +function onCopied(err) { + if (err) { + var separator = new Array(81).join('!'); + + console.error(separator); + console.error( + 'Failed to copy `' + NPM_SHRINKWRAP_FILE + '` to `' + NPM_SHRINKWRAP_CACHED_FILE + '`:'); + console.error(err); + console.error(separator); + } +}