From 2b7da188aeb320af7a2cefa6ab1825408f965684 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Thu, 27 Apr 2017 13:47:42 -0700 Subject: [PATCH 01/28] update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48df6ce83dbe..8f3d05ea72c4 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,4 @@ lighthouse-cli/types/*.map lighthouse-core/report/partials/templates/ lighthouse-core/report/templates/*.js -plots/out/ +plots/out** From 5577cf74a8f9eaa07a2c985e2d90afdebb279826 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Thu, 27 Apr 2017 13:48:00 -0700 Subject: [PATCH 02/28] add flags for mobile measurement --- plots/ab-screenshot/analyze.js | 33 +++++++++++++++++++++++---------- plots/measure.js | 27 ++++++++++++++++++++------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index d02eb5b37f6c..bc86e3c9c86e 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -22,6 +22,8 @@ const path = require('path'); const opn = require('opn'); const args = require('yargs').argv; +const RUNS = args.runs || 1; + const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events'); const constants = require('../constants'); @@ -84,25 +86,36 @@ function aggregate(outPathA, outPathB) { if (!utils.isDir(sitePathB)) { return; } - const siteScreenshotsComparison = { - siteName: siteDir, - runA: analyzeSingleRunScreenshots(sitePathA), - runB: analyzeSingleRunScreenshots(sitePathB) - }; - results.push(siteScreenshotsComparison); + + for (let i = 0; i < RUNS; i++) { + const runDirA = getRunDir(sitePathA, i); + const runDirB = getRunDir(sitePathB, i); + const siteScreenshotsComparison = { + siteName: `${siteDir} runA: ${runDirA} runB: ${runDirB}`, + runA: analyzeSingleRunScreenshots(path.resolve(sitePathA, runDirA)), + runB: analyzeSingleRunScreenshots(path.resolve(sitePathB, runDirB)), + }; + results.push(siteScreenshotsComparison); + } }); return results; } /** - * Analyzes the screenshots for the first run of a particular site. * @param {string} sitePath + * @param {number} runIndex + */ +function getRunDir(sitePath, runIndex) { + return sortAndFilterRunFolders(fs.readdirSync(sitePath))[runIndex]; +} + +/** + * Analyzes the screenshots for the first run of a particular site. + * @param {string} runPath * @return {!SingleRunScreenshots} */ -function analyzeSingleRunScreenshots(sitePath) { - const runDir = sortAndFilterRunFolders(fs.readdirSync(sitePath))[0]; - const runPath = path.resolve(sitePath, runDir); +function analyzeSingleRunScreenshots(runPath) { const lighthouseResultsPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const lighthouseResults = JSON.parse(fs.readFileSync(lighthouseResultsPath)); diff --git a/plots/measure.js b/plots/measure.js index dfe298bc43cd..409de01a8ab9 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -20,6 +20,7 @@ const path = require('path'); const parseURL = require('url').parse; const mkdirp = require('mkdirp'); +const args = require('yargs').argv; const constants = require('./constants.js'); const utils = require('./utils.js'); @@ -29,9 +30,21 @@ const ChromeLauncher = require('../lighthouse-cli/chrome-launcher.js').ChromeLau const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const NUMBER_OF_RUNS = 20; - -const URLS = [ +const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; +const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling'] ; +const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling'] ; +const KEEP_FIRST_RUN = args['keep-first-run']; +const NUMBER_OF_RUNS = args.n || 20; + +const FLAGS = { + output: 'json', + disableCpuThrottling: DISABLE_CPU_THROTTLING, + disableNetworkThrottling: DISABLE_NETWORK_THROTTLING, + disableDeviceEmulation: DISABLE_DEVICE_EMULATION, +}; +console.log('Running lighthouse with flags: ', FLAGS); + +const URLS = args.site ? [args.site] : [ // Flagship sites 'https://nytimes.com', 'https://flipkart.com', @@ -162,8 +175,9 @@ function runAnalysis() { const id = i.toString(); const isFirstRun = i === 0; + const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; for (const url of URLS) { - promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun: isFirstRun})); + promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun})); } } return promise; @@ -199,8 +213,7 @@ function singleRunAnalysis(url, id, {ignoreRun}) { * @return {!Promise} */ function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { - const flags = {output: 'json'}; - return lighthouse(url, flags, config) + return lighthouse(url, FLAGS, config) .then(lighthouseResults => { if (ignoreRun) { return; @@ -209,7 +222,7 @@ function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { .saveAssets(lighthouseResults.artifacts, lighthouseResults.audits, assetsPath) .then(() => { lighthouseResults.artifacts = undefined; - return Printer.write(lighthouseResults, flags.output, outputPath); + return Printer.write(lighthouseResults, FLAGS.output, outputPath); }); }) .catch(err => console.error(err)); // eslint-disable-line no-console From 9d11132fa2342ac8613895403535059610b234a4 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 2 May 2017 14:47:02 -0700 Subject: [PATCH 03/28] top 18 sites --- plots/measure.js | 176 ++++++++++++++++++++++++----------------------- 1 file changed, 89 insertions(+), 87 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 409de01a8ab9..716974f6a794 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -31,8 +31,8 @@ const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; -const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling'] ; -const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling'] ; +const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling']; +const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling']; const KEEP_FIRST_RUN = args['keep-first-run']; const NUMBER_OF_RUNS = args.n || 20; @@ -50,84 +50,85 @@ const URLS = args.site ? [args.site] : [ 'https://flipkart.com', 'http://www.espn.com/', 'https://www.washingtonpost.com/pwa/', + 'https://www.airbnb.com/', // TTI Tester sites - 'https://housing.com/in/buy/real-estate-hyderabad', - 'http://www.npr.org/', - 'http://www.vevo.com/', - 'https://weather.com/', + // 'https://housing.com/in/buy/real-estate-hyderabad', + // 'http://www.npr.org/', + // 'http://www.vevo.com/', + // 'https://weather.com/', 'https://www.nasa.gov/', - 'https://vine.co/', + // 'https://vine.co/', 'http://www.booking.com/', - 'http://www.thestar.com.my', - 'http://www.58pic.com', - 'http://www.dawn.com/', - 'https://www.ebs.in/IPS/', - - // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites - // (http://www.alexa.com/topsites) - // Removed adult websites and duplicates (e.g. google int'l websites) - // Also removed sites that don't have significant index pages: - // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", - // "live.com", "adf.ly", "googleusercontent.com", - - 'https://google.com', + // 'http://www.thestar.com.my', + // 'http://www.58pic.com', + // 'http://www.dawn.com/', + // 'https://www.ebs.in/IPS/', + + // // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites + // // (http://www.alexa.com/topsites) + // // Removed adult websites and duplicates (e.g. google int'l websites) + // // Also removed sites that don't have significant index pages: + // // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", + // // "live.com", "adf.ly", "googleusercontent.com", + + 'https://www.google.com/search?q=flowers', 'https://youtube.com', - 'https://facebook.com', - 'https://baidu.com', - 'https://wikipedia.org', - 'https://yahoo.com', + // 'https://facebook.com', + // 'https://baidu.com', + 'https://en.wikipedia.org/wiki/Google', + // 'https://yahoo.com', 'https://amazon.com', - 'http://www.qq.com/', - 'https://taobao.com', - 'https://vk.com', - 'https://twitter.com', - 'https://instagram.com', - 'http://www.hao123.cn/', - 'http://www.sohu.com/', - 'https://sina.com.cn', + // 'http://www.qq.com/', + // 'https://taobao.com', + // 'https://vk.com', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + // 'http://www.hao123.cn/', + // 'http://www.sohu.com/', + // 'https://sina.com.cn', 'https://reddit.com', - 'https://linkedin.com', - 'https://tmall.com', - 'https://weibo.com', - 'https://360.cn', - 'https://yandex.ru', + // 'https://linkedin.com', + // 'https://tmall.com', + // 'https://weibo.com', + // 'https://360.cn', + // 'https://yandex.ru', 'https://ebay.com', - 'https://bing.com', - 'https://msn.com', - 'https://www.sogou.com/', - 'https://wordpress.com', - 'https://microsoft.com', - 'https://tumblr.com', - 'https://aliexpress.com', - 'https://blogspot.com', - 'https://netflix.com', - 'https://ok.ru', + // 'https://bing.com', + // 'https://msn.com', + // 'https://www.sogou.com/', + // 'https://wordpress.com', + // 'https://microsoft.com', + // 'https://tumblr.com', + // 'https://aliexpress.com', + // 'https://blogspot.com', + // 'https://netflix.com', + // 'https://ok.ru', 'https://stackoverflow.com', - 'https://imgur.com', + // 'https://imgur.com', 'https://apple.com', - 'http://www.naver.com/', - 'https://mail.ru', - 'http://www.imdb.com/', - 'https://office.com', - 'https://github.com', - 'https://pinterest.com', - 'https://paypal.com', - 'http://www.tianya.cn/', - 'https://diply.com', - 'https://twitch.tv', - 'https://adobe.com', - 'https://wikia.com', - 'https://coccoc.com', - 'https://so.com', - 'https://fc2.com', - 'https://www.pixnet.net/', - 'https://dropbox.com', - 'https://zhihu.com', - 'https://whatsapp.com', - 'https://alibaba.com', - 'https://ask.com', - 'https://bbc.com' + // 'http://www.naver.com/', + // 'https://mail.ru', + // 'http://www.imdb.com/', + // 'https://office.com', + // 'https://github.com', + // 'https://pinterest.com', + // 'https://paypal.com', + // 'http://www.tianya.cn/', + // 'https://diply.com', + // 'https://twitch.tv', + // 'https://adobe.com', + // 'https://wikia.com', + // 'https://coccoc.com', + // 'https://so.com', + // 'https://fc2.com', + // 'https://www.pixnet.net/', + // 'https://dropbox.com', + // 'https://zhihu.com', + // 'https://whatsapp.com', + // 'https://alibaba.com', + // 'https://ask.com', + // 'https://bbc.com' ]; /** @@ -143,18 +144,7 @@ function main() { return; } - const launcher = new ChromeLauncher(); - launcher - .isDebuggerReady() - .catch(() => launcher.run()) - .then(() => runAnalysis()) - .then(() => launcher.kill()) - .catch(err => launcher.kill().then( - () => { - throw err; - }, - console.error // eslint-disable-line no-console - )); + runAnalysisWithNewChromeInstances() } main(); @@ -163,7 +153,7 @@ main(); * Returns a promise chain that analyzes all the sites n times. * @return {!Promise} */ -function runAnalysis() { +function runAnalysisWithNewChromeInstances() { let promise = Promise.resolve(); // Running it n + 1 times because the first run is deliberately ignored @@ -177,7 +167,19 @@ function runAnalysis() { const isFirstRun = i === 0; const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; for (const url of URLS) { - promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun})); + promise = promise.then(() => { + const launcher = new ChromeLauncher(); + return launcher.isDebuggerReady() + .catch(() => launcher.run()) + .then(() => singleRunAnalysis(url, id, { ignoreRun })) + .then(() => launcher.kill()) + .catch(err => launcher.kill().then( + () => { + throw err; + }, + console.error // eslint-disable-line no-console + )); + }); } } return promise; @@ -190,7 +192,7 @@ function runAnalysis() { * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function singleRunAnalysis(url, id, {ignoreRun}) { +function singleRunAnalysis(url, id, { ignoreRun }) { console.log('Measuring site:', url, 'run:', id); // eslint-disable-line no-console const parsedURL = parseURL(url); const urlBasedFilename = sanitizeURL(`${parsedURL.host}-${parsedURL.pathname}`); @@ -200,7 +202,7 @@ function singleRunAnalysis(url, id, {ignoreRun}) { } const outputPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const assetsPath = path.resolve(runPath, 'assets'); - return analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}); + return analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun }); } /** @@ -212,7 +214,7 @@ function singleRunAnalysis(url, id, {ignoreRun}) { * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { +function analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun }) { return lighthouse(url, FLAGS, config) .then(lighthouseResults => { if (ignoreRun) { From 4655cbecb20c61e8f61a7280cfc472da69a157e5 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 2 May 2017 15:37:00 -0700 Subject: [PATCH 04/28] remove airbnb - not reliable --- plots/measure.js | 1 - 1 file changed, 1 deletion(-) diff --git a/plots/measure.js b/plots/measure.js index 716974f6a794..3a0897358722 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -50,7 +50,6 @@ const URLS = args.site ? [args.site] : [ 'https://flipkart.com', 'http://www.espn.com/', 'https://www.washingtonpost.com/pwa/', - 'https://www.airbnb.com/', // TTI Tester sites // 'https://housing.com/in/buy/real-estate-hyderabad', From d38d070f3132daceb5c958f84ec4ce2dc63f2f90 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 2 May 2017 17:06:43 -0700 Subject: [PATCH 05/28] adjust measure.js --- plots/measure.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 3a0897358722..718ce715cee8 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -34,7 +34,11 @@ const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling']; const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling']; const KEEP_FIRST_RUN = args['keep-first-run']; -const NUMBER_OF_RUNS = args.n || 20; + +// Running it n + 1 times if the first run is deliberately ignored +// because it has different perf characteristics from subsequent runs +// (e.g. DNS cache which can't be easily reset between runs) +const NUMBER_OF_RUNS = (args.n || 20) + (KEEP_FIRST_RUN ? 0 : 1); const FLAGS = { output: 'json', @@ -53,7 +57,7 @@ const URLS = args.site ? [args.site] : [ // TTI Tester sites // 'https://housing.com/in/buy/real-estate-hyderabad', - // 'http://www.npr.org/', + 'http://www.npr.org/', // 'http://www.vevo.com/', // 'https://weather.com/', 'https://www.nasa.gov/', @@ -155,10 +159,7 @@ main(); function runAnalysisWithNewChromeInstances() { let promise = Promise.resolve(); - // Running it n + 1 times because the first run is deliberately ignored - // because it has different perf characteristics from subsequent runs - // (e.g. DNS cache which can't be easily reset between runs) - for (let i = 0; i <= NUMBER_OF_RUNS; i++) { + for (let i = 0; i < NUMBER_OF_RUNS; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); From a7eaf8e93f294eb98e35bd8547d15f63d3ab8d17 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Wed, 3 May 2017 11:33:02 -0700 Subject: [PATCH 06/28] define SUBSET --subset flag --- plots/measure.js | 132 +++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 56 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 718ce715cee8..8c275ebb9df4 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -48,7 +48,27 @@ const FLAGS = { }; console.log('Running lighthouse with flags: ', FLAGS); -const URLS = args.site ? [args.site] : [ +const SUBSET = [ + 'https://en.wikipedia.org/wiki/Google', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'https://amazon.com', + 'https://nytimes.com', + 'https://www.google.com/search?q=flowers', + + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + 'http://www.npr.org/', + 'https://www.nasa.gov/', + 'http://www.booking.com/', + 'https://youtube.com', + 'https://reddit.com', + 'https://ebay.com', + 'https://stackoverflow.com', + 'https://apple.com', +]; +const URLS = args.subset ? SUBSET : args.site ? [args.site] : [ // Flagship sites 'https://nytimes.com', 'https://flipkart.com', @@ -56,17 +76,17 @@ const URLS = args.site ? [args.site] : [ 'https://www.washingtonpost.com/pwa/', // TTI Tester sites - // 'https://housing.com/in/buy/real-estate-hyderabad', + 'https://housing.com/in/buy/real-estate-hyderabad', 'http://www.npr.org/', - // 'http://www.vevo.com/', - // 'https://weather.com/', + 'http://www.vevo.com/', + 'https://weather.com/', 'https://www.nasa.gov/', - // 'https://vine.co/', + 'https://vine.co/', 'http://www.booking.com/', - // 'http://www.thestar.com.my', - // 'http://www.58pic.com', - // 'http://www.dawn.com/', - // 'https://www.ebs.in/IPS/', + 'http://www.thestar.com.my', + 'http://www.58pic.com', + 'http://www.dawn.com/', + 'https://www.ebs.in/IPS/', // // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites // // (http://www.alexa.com/topsites) @@ -77,61 +97,61 @@ const URLS = args.site ? [args.site] : [ 'https://www.google.com/search?q=flowers', 'https://youtube.com', - // 'https://facebook.com', - // 'https://baidu.com', + 'https://facebook.com', + 'https://baidu.com', 'https://en.wikipedia.org/wiki/Google', - // 'https://yahoo.com', + 'https://yahoo.com', 'https://amazon.com', - // 'http://www.qq.com/', - // 'https://taobao.com', - // 'https://vk.com', + 'http://www.qq.com/', + 'https://taobao.com', + 'https://vk.com', 'https://mobile.twitter.com/ChromeDevTools', 'https://www.instagram.com/stephencurry30', - // 'http://www.hao123.cn/', - // 'http://www.sohu.com/', - // 'https://sina.com.cn', + 'http://www.hao123.cn/', + 'http://www.sohu.com/', + 'https://sina.com.cn', 'https://reddit.com', - // 'https://linkedin.com', - // 'https://tmall.com', - // 'https://weibo.com', - // 'https://360.cn', - // 'https://yandex.ru', + 'https://linkedin.com', + 'https://tmall.com', + 'https://weibo.com', + 'https://360.cn', + 'https://yandex.ru', 'https://ebay.com', - // 'https://bing.com', - // 'https://msn.com', - // 'https://www.sogou.com/', - // 'https://wordpress.com', - // 'https://microsoft.com', - // 'https://tumblr.com', - // 'https://aliexpress.com', - // 'https://blogspot.com', - // 'https://netflix.com', - // 'https://ok.ru', + 'https://bing.com', + 'https://msn.com', + 'https://www.sogou.com/', + 'https://wordpress.com', + 'https://microsoft.com', + 'https://tumblr.com', + 'https://aliexpress.com', + 'https://blogspot.com', + 'https://netflix.com', + 'https://ok.ru', 'https://stackoverflow.com', - // 'https://imgur.com', + 'https://imgur.com', 'https://apple.com', - // 'http://www.naver.com/', - // 'https://mail.ru', - // 'http://www.imdb.com/', - // 'https://office.com', - // 'https://github.com', - // 'https://pinterest.com', - // 'https://paypal.com', - // 'http://www.tianya.cn/', - // 'https://diply.com', - // 'https://twitch.tv', - // 'https://adobe.com', - // 'https://wikia.com', - // 'https://coccoc.com', - // 'https://so.com', - // 'https://fc2.com', - // 'https://www.pixnet.net/', - // 'https://dropbox.com', - // 'https://zhihu.com', - // 'https://whatsapp.com', - // 'https://alibaba.com', - // 'https://ask.com', - // 'https://bbc.com' + 'http://www.naver.com/', + 'https://mail.ru', + 'http://www.imdb.com/', + 'https://office.com', + 'https://github.com', + 'https://pinterest.com', + 'https://paypal.com', + 'http://www.tianya.cn/', + 'https://diply.com', + 'https://twitch.tv', + 'https://adobe.com', + 'https://wikia.com', + 'https://coccoc.com', + 'https://so.com', + 'https://fc2.com', + 'https://www.pixnet.net/', + 'https://dropbox.com', + 'https://zhihu.com', + 'https://whatsapp.com', + 'https://alibaba.com', + 'https://ask.com', + 'https://bbc.com' ]; /** From adb59a9d6d52e7ac8fadba387055f7cd52ddfa5f Mon Sep 17 00:00:00 2001 From: Will Chen Date: Thu, 4 May 2017 10:45:17 -0700 Subject: [PATCH 07/28] fixup measure --- plots/measure.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 8c275ebb9df4..57139af3a4c1 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -30,9 +30,9 @@ const ChromeLauncher = require('../lighthouse-cli/chrome-launcher.js').ChromeLau const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; -const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling']; -const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling']; +const DISABLE_DEVICE_EMULATION = false; +const DISABLE_CPU_THROTTLING = true; +const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling'] || false; const KEEP_FIRST_RUN = args['keep-first-run']; // Running it n + 1 times if the first run is deliberately ignored @@ -46,7 +46,7 @@ const FLAGS = { disableNetworkThrottling: DISABLE_NETWORK_THROTTLING, disableDeviceEmulation: DISABLE_DEVICE_EMULATION, }; -console.log('Running lighthouse with flags: ', FLAGS); +console.log('Running lighthouse with flag\n disableNetworkThrottling: ', FLAGS.disableNetworkThrottling); const SUBSET = [ 'https://en.wikipedia.org/wiki/Google', From 57b763e182292551e1ea04a6efe4d1ef981ea728 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Fri, 5 May 2017 13:52:10 -0700 Subject: [PATCH 08/28] make screenshot/analyze.js handle missing results --- plots/ab-screenshot/analyze.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index bc86e3c9c86e..c87f681d7c68 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -90,10 +90,21 @@ function aggregate(outPathA, outPathB) { for (let i = 0; i < RUNS; i++) { const runDirA = getRunDir(sitePathA, i); const runDirB = getRunDir(sitePathB, i); + + const runPathA = path.resolve(sitePathA, runDirA); + const runPathB = path.resolve(sitePathB, runDirB); + + const lighthouseFileA = path.resolve(runPathA, constants.LIGHTHOUSE_RESULTS_FILENAME); + const lighthouseFileB = path.resolve(runPathB, constants.LIGHTHOUSE_RESULTS_FILENAME); + + if (!utils.isFile(lighthouseFileA) || !utils.isFile(lighthouseFileB)) { + continue; + } + const siteScreenshotsComparison = { siteName: `${siteDir} runA: ${runDirA} runB: ${runDirB}`, - runA: analyzeSingleRunScreenshots(path.resolve(sitePathA, runDirA)), - runB: analyzeSingleRunScreenshots(path.resolve(sitePathB, runDirB)), + runA: analyzeSingleRunScreenshots(runPathA), + runB: analyzeSingleRunScreenshots(runPathB), }; results.push(siteScreenshotsComparison); } From 506dc30944d66a8f887486b05c70b7d5800e927e Mon Sep 17 00:00:00 2001 From: Will Chen Date: Fri, 5 May 2017 13:55:08 -0700 Subject: [PATCH 09/28] adjustment factors --- lighthouse-core/lib/emulation.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lighthouse-core/lib/emulation.js b/lighthouse-core/lib/emulation.js index a8708046bd44..7ea152bfba37 100644 --- a/lighthouse-core/lib/emulation.js +++ b/lighthouse-core/lib/emulation.js @@ -41,10 +41,13 @@ const NEXUS5X_USERAGENT = { '(KHTML, like Gecko) Chrome/59.0.3033.0 Mobile Safari/537.36' }; +const LATENCY_FACTOR = 3.75; +const THROUGHPUT_FACTOR = 1.1; + const TYPICAL_MOBILE_THROTTLING_METRICS = { - latency: 150, // 150ms - downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8), // 1.6Mbps - uploadThroughput: Math.floor(750 * 1024 / 8), // 750Kbps + latency: 150 * LATENCY_FACTOR, // 150ms + downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8) / THROUGHPUT_FACTOR, // 1.6Mbps + uploadThroughput: Math.floor(750 * 1024 / 8) / THROUGHPUT_FACTOR, // 750Kbps offline: false }; From b8c4577d58d709ce2de84659302ab960c7dbb1ac Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 18 Apr 2017 11:35:40 -0700 Subject: [PATCH 10/28] cherry-picked plot-per-site --- plots/plot-per-site.html | 29 ++++++++++++++ plots/plot-per-site.js | 86 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 plots/plot-per-site.html create mode 100644 plots/plot-per-site.js diff --git a/plots/plot-per-site.html b/plots/plot-per-site.html new file mode 100644 index 000000000000..ee68574ae156 --- /dev/null +++ b/plots/plot-per-site.html @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + diff --git a/plots/plot-per-site.js b/plots/plot-per-site.js new file mode 100644 index 000000000000..50f6fcfb9fcd --- /dev/null +++ b/plots/plot-per-site.js @@ -0,0 +1,86 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/* global Plotly, generatedResults */ +/* eslint-env browser */ + +const IGNORED_METRICS = new Set(['Navigation Start']); + +const metrics = Object.keys(generatedResults).filter(metric => !IGNORED_METRICS.has(metric)); + +let elementId = 1; + +/** + * Incrementally renders the plot, otherwise it hangs the browser + * because it's generating so many charts. + */ +const queuedPlots = []; + +function enqueuePlot(fn) { + const isFirst = queuedPlots.length == 0; + queuedPlots.push(fn); + if (isFirst) { + renderPlots(); + } +} + +function renderPlots() { + window.requestAnimationFrame(_ => { + const plotFn = queuedPlots.shift(); + if (plotFn) { + plotFn(); + renderPlots(); + } + }); +} + +function createChartElement(height = 800) { + const div = document.createElement('div'); + div.style = `width: 100%; height: ${height}px`; + div.id = 'chart' + elementId++; + document.body.appendChild(div); + return div.id; +} + +function generateGroupedBarChart() { + const sitesCount = metrics.reduce( + (acc, metric) => Math.max(acc, generatedResults[metric].length), + 0 + ); + for (let i = 0; i < sitesCount; i++) { + const data = metrics.map(metric => ({ + y: generatedResults[metric][i].metrics.map(m => m ? m.timing : null), + name: metric, + type: 'bar' + })); + + const layout = { + yaxis: { + rangemode: 'tozero' + }, + hovermode: 'closest', + barmode: 'group', + title: generatedResults[metrics[0]][i].site + }; + enqueuePlot(_ => { + Plotly.newPlot(createChartElement(), data, layout); + }); + } +} + +generateGroupedBarChart(); From ed447058fd3708976bbef2a01f28f5ed48415747 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Mon, 8 May 2017 14:32:13 -0700 Subject: [PATCH 11/28] wrap --- plots/measure.js | 4 +++- plots/utils.js | 60 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 57139af3a4c1..24593a987761 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -60,13 +60,15 @@ const SUBSET = [ 'http://www.espn.com/', 'https://www.washingtonpost.com/pwa/', 'http://www.npr.org/', - 'https://www.nasa.gov/', 'http://www.booking.com/', 'https://youtube.com', 'https://reddit.com', 'https://ebay.com', 'https://stackoverflow.com', 'https://apple.com', + + // Could not run nasa on gin3g + // 'https://www.nasa.gov/', ]; const URLS = args.subset ? SUBSET : args.site ? [args.site] : [ // Flagship sites diff --git a/plots/utils.js b/plots/utils.js index 5bd2b8203091..cb280e458ce6 100644 --- a/plots/utils.js +++ b/plots/utils.js @@ -17,6 +17,7 @@ 'use strict'; const fs = require('fs'); +const path = require('path'); /** * @param {string} path @@ -52,8 +53,65 @@ function shuffle(array) { } } +/** + * @param {string} src + * @param {string} dest + */ +function copyRecursive(src, dest) { + try { + if (isFile(src)) { + copy(src, dest); + return; + } + var targetDirPath = path.resolve(dest, path.basename(src)); + if (!fs.existsSync(targetDirPath)) + fs.mkdirSync(targetDirPath); + if (isDir(src)) { + var files = fs.readdirSync(src); + for (var i = 0; i < files.length; i++) { + var childPath = path.resolve(src, files[i]); + if (isDir(childPath)) { + copyRecursive(childPath, targetDirPath); + } else { + var targetFilePath = path.resolve(targetDirPath, path.basename(childPath)); + fs.writeFileSync(targetFilePath, fs.readFileSync(childPath)); + } + } + } + } catch (error) { + throw new Error(`Received an error: [${error}] while trying to copy: ${src} -> ${dest}`); + } +} + +/** + * @param {string} filePath + */ +function removeRecursive(filePath) { + try { + if (fs.existsSync(filePath)) { + if (isFile(filePath)) { + fs.unlinkSync(filePath); + return; + } + var files = fs.readdirSync(filePath); + for (var i = 0; i < files.length; i++) { + var childPath = path.resolve(filePath, files[i]); + if (isDir(childPath)) + removeRecursive(childPath); + else + fs.unlinkSync(childPath); + } + fs.rmdirSync(filePath); + } + } catch (error) { + throw new Error(`Received an error: [${error}] while trying to remove: ${filePath}`); + } +} + module.exports = { isDir, isFile, - shuffle + shuffle, + copyRecursive, + removeRecursive, }; From 39f6b4c2b5955bb305ef173cb61dde6981be8557 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 9 May 2017 11:06:29 -0700 Subject: [PATCH 12/28] fix comments --- plots/measure.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 24593a987761..07cefd68f41b 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -90,12 +90,12 @@ const URLS = args.subset ? SUBSET : args.site ? [args.site] : [ 'http://www.dawn.com/', 'https://www.ebs.in/IPS/', - // // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites - // // (http://www.alexa.com/topsites) - // // Removed adult websites and duplicates (e.g. google int'l websites) - // // Also removed sites that don't have significant index pages: - // // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", - // // "live.com", "adf.ly", "googleusercontent.com", + // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites + // (http://www.alexa.com/topsites) + // Removed adult websites and duplicates (e.g. google int'l websites) + // Also removed sites that don't have significant index pages: + // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", + // "live.com", "adf.ly", "googleusercontent.com", 'https://www.google.com/search?q=flowers', 'https://youtube.com', From 5338f898a18b75a5403861f805505c633adafc7f Mon Sep 17 00:00:00 2001 From: Will Chen Date: Mon, 15 May 2017 11:58:24 -0700 Subject: [PATCH 13/28] update --- plots/measure.js | 161 +++++++++++++++++++++++++++++++++++------------ plots/utils.js | 37 +---------- 2 files changed, 123 insertions(+), 75 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 07cefd68f41b..a401063f07bd 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -20,7 +20,30 @@ const path = require('path'); const parseURL = require('url').parse; const mkdirp = require('mkdirp'); -const args = require('yargs').argv; +const args = require('yargs') + .help('help') + .usage('npm run measure -- [options]') + .describe({ + 'n': 'Number of runs to do per site', + 'reuse-chrome': 'Reuse the same Chrome instance across all site runs', + 'keep-first-run': 'By default if you use --reuse-chrome, the first run results are discarded', + }) + .group( + ['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling'], + 'Chrome DevTools settings:') + .describe({ + 'disable-device-emulation': 'Disable Nexus 5X emulation', + 'disable-cpu-throttling': 'Disable CPU throttling', + 'disable-network-throttling': 'Disable network throttling', + }) + .group(['sites-path', 'subset', 'site'], 'Options to specify sites:') + .describe({ + 'sites-path': 'Include relative path of a json file with urls to run', + 'subset': 'Measure a subset of popular sites', + 'site': 'Include a specific site URL to run', + }) + .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) + .argv; const constants = require('./constants.js'); const utils = require('./utils.js'); @@ -30,15 +53,20 @@ const ChromeLauncher = require('../lighthouse-cli/chrome-launcher.js').ChromeLau const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const DISABLE_DEVICE_EMULATION = false; -const DISABLE_CPU_THROTTLING = true; -const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling'] || false; -const KEEP_FIRST_RUN = args['keep-first-run']; +const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; +const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling']; +const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling']; +const REUSE_CHROME = args['reuse-chrome']; +const KEEP_FIRST_RUN = args['keep-first-run'] || !REUSE_CHROME; +const SITES_PATH = args['sites-path']; +const SUBSET = args['subset']; +const SITE = args['site']; +const CUSTOM_NUMBER_OF_RUNS = args['n']; // Running it n + 1 times if the first run is deliberately ignored // because it has different perf characteristics from subsequent runs // (e.g. DNS cache which can't be easily reset between runs) -const NUMBER_OF_RUNS = (args.n || 20) + (KEEP_FIRST_RUN ? 0 : 1); +const NUMBER_OF_RUNS = (CUSTOM_NUMBER_OF_RUNS || 20) + (KEEP_FIRST_RUN ? 0 : 1); const FLAGS = { output: 'json', @@ -46,31 +74,8 @@ const FLAGS = { disableNetworkThrottling: DISABLE_NETWORK_THROTTLING, disableDeviceEmulation: DISABLE_DEVICE_EMULATION, }; -console.log('Running lighthouse with flag\n disableNetworkThrottling: ', FLAGS.disableNetworkThrottling); -const SUBSET = [ - 'https://en.wikipedia.org/wiki/Google', - 'https://mobile.twitter.com/ChromeDevTools', - 'https://www.instagram.com/stephencurry30', - 'https://amazon.com', - 'https://nytimes.com', - 'https://www.google.com/search?q=flowers', - - 'https://flipkart.com', - 'http://www.espn.com/', - 'https://www.washingtonpost.com/pwa/', - 'http://www.npr.org/', - 'http://www.booking.com/', - 'https://youtube.com', - 'https://reddit.com', - 'https://ebay.com', - 'https://stackoverflow.com', - 'https://apple.com', - - // Could not run nasa on gin3g - // 'https://www.nasa.gov/', -]; -const URLS = args.subset ? SUBSET : args.site ? [args.site] : [ +const SITES = [ // Flagship sites 'https://nytimes.com', 'https://flipkart.com', @@ -156,12 +161,45 @@ const URLS = args.subset ? SUBSET : args.site ? [args.site] : [ 'https://bbc.com' ]; -/** - * Launches Chrome once at the beginning, runs all the analysis, - * and then kills Chrome. - * TODO(chenwilliam): measure the overhead of starting chrome, if it's minimal - * then open a fresh Chrome instance for each run. - */ +function getUrls() { + if (SITES_PATH) { + return require(path.resolve(__dirname, SITES_PATH)); + } + + if (SITE) { + return [SITE]; + } + + if (SUBSET) { + return [ + 'https://en.wikipedia.org/wiki/Google', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'https://amazon.com', + 'https://nytimes.com', + 'https://www.google.com/search?q=flowers', + + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + 'http://www.npr.org/', + 'http://www.booking.com/', + 'https://youtube.com', + 'https://reddit.com', + 'https://ebay.com', + 'https://stackoverflow.com', + 'https://apple.com', + + // Could not run nasa on gin3g + 'https://www.nasa.gov/', + ]; + } + + return SITES; +} + +const URLS = getUrls(); + function main() { if (utils.isDir(constants.OUT_PATH)) { console.log('ERROR: Found output from previous run at: ', constants.OUT_PATH); // eslint-disable-line no-console @@ -169,12 +207,28 @@ function main() { return; } - runAnalysisWithNewChromeInstances() + if (REUSE_CHROME) { + const launcher = new ChromeLauncher(); + launcher + .isDebuggerReady() + .catch(() => launcher.run()) + .then(() => runAnalysisWithExistingChromeInstances()) + .then(() => launcher.kill()) + .catch(err => launcher.kill().then( + () => { + throw err; + }, + console.error // eslint-disable-line no-console + )); + return; + } + runAnalysisWithNewChromeInstances(); } main(); /** + * Launches a new Chrome instance for each site run. * Returns a promise chain that analyzes all the sites n times. * @return {!Promise} */ @@ -193,7 +247,7 @@ function runAnalysisWithNewChromeInstances() { const launcher = new ChromeLauncher(); return launcher.isDebuggerReady() .catch(() => launcher.run()) - .then(() => singleRunAnalysis(url, id, { ignoreRun })) + .then(() => singleRunAnalysis(url, id, {ignoreRun})) .then(() => launcher.kill()) .catch(err => launcher.kill().then( () => { @@ -207,6 +261,31 @@ function runAnalysisWithNewChromeInstances() { return promise; } +/** + * Reuses existing Chrome instance for all site runs. + * Returns a promise chain that analyzes all the sites n times. + * @return {!Promise} + */ +function runAnalysisWithExistingChromeInstances() { + let promise = Promise.resolve(); + + // Running it n + 1 times because the first run is deliberately ignored + // because it has different perf characteristics from subsequent runs + // (e.g. DNS cache which can't be easily reset between runs) + for (let i = 0; i <= NUMBER_OF_RUNS; i++) { + // Averages out any order-dependent effects such as memory pressure + utils.shuffle(URLS); + + const id = i.toString(); + const isFirstRun = i === 0; + const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; + for (const url of URLS) { + promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun})); + } + } + return promise; +} + /** * Analyzes a site a single time using lighthouse. * @param {string} url @@ -214,7 +293,7 @@ function runAnalysisWithNewChromeInstances() { * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function singleRunAnalysis(url, id, { ignoreRun }) { +function singleRunAnalysis(url, id, {ignoreRun}) { console.log('Measuring site:', url, 'run:', id); // eslint-disable-line no-console const parsedURL = parseURL(url); const urlBasedFilename = sanitizeURL(`${parsedURL.host}-${parsedURL.pathname}`); @@ -224,7 +303,7 @@ function singleRunAnalysis(url, id, { ignoreRun }) { } const outputPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const assetsPath = path.resolve(runPath, 'assets'); - return analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun }); + return analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}); } /** @@ -236,7 +315,7 @@ function singleRunAnalysis(url, id, { ignoreRun }) { * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun }) { +function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { return lighthouse(url, FLAGS, config) .then(lighthouseResults => { if (ignoreRun) { diff --git a/plots/utils.js b/plots/utils.js index cb280e458ce6..68b450d88a5f 100644 --- a/plots/utils.js +++ b/plots/utils.js @@ -53,36 +53,6 @@ function shuffle(array) { } } -/** - * @param {string} src - * @param {string} dest - */ -function copyRecursive(src, dest) { - try { - if (isFile(src)) { - copy(src, dest); - return; - } - var targetDirPath = path.resolve(dest, path.basename(src)); - if (!fs.existsSync(targetDirPath)) - fs.mkdirSync(targetDirPath); - if (isDir(src)) { - var files = fs.readdirSync(src); - for (var i = 0; i < files.length; i++) { - var childPath = path.resolve(src, files[i]); - if (isDir(childPath)) { - copyRecursive(childPath, targetDirPath); - } else { - var targetFilePath = path.resolve(targetDirPath, path.basename(childPath)); - fs.writeFileSync(targetFilePath, fs.readFileSync(childPath)); - } - } - } - } catch (error) { - throw new Error(`Received an error: [${error}] while trying to copy: ${src} -> ${dest}`); - } -} - /** * @param {string} filePath */ @@ -93,9 +63,9 @@ function removeRecursive(filePath) { fs.unlinkSync(filePath); return; } - var files = fs.readdirSync(filePath); - for (var i = 0; i < files.length; i++) { - var childPath = path.resolve(filePath, files[i]); + const files = fs.readdirSync(filePath); + for (let i = 0; i < files.length; i++) { + const childPath = path.resolve(filePath, files[i]); if (isDir(childPath)) removeRecursive(childPath); else @@ -112,6 +82,5 @@ module.exports = { isDir, isFile, shuffle, - copyRecursive, removeRecursive, }; From d6cabdbb954edd95a3eec37bf6fb0f2b62d337f0 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Mon, 15 May 2017 11:59:38 -0700 Subject: [PATCH 14/28] undo emulation.js --- lighthouse-core/lib/emulation.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lighthouse-core/lib/emulation.js b/lighthouse-core/lib/emulation.js index 7ea152bfba37..a8708046bd44 100644 --- a/lighthouse-core/lib/emulation.js +++ b/lighthouse-core/lib/emulation.js @@ -41,13 +41,10 @@ const NEXUS5X_USERAGENT = { '(KHTML, like Gecko) Chrome/59.0.3033.0 Mobile Safari/537.36' }; -const LATENCY_FACTOR = 3.75; -const THROUGHPUT_FACTOR = 1.1; - const TYPICAL_MOBILE_THROTTLING_METRICS = { - latency: 150 * LATENCY_FACTOR, // 150ms - downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8) / THROUGHPUT_FACTOR, // 1.6Mbps - uploadThroughput: Math.floor(750 * 1024 / 8) / THROUGHPUT_FACTOR, // 750Kbps + latency: 150, // 150ms + downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8), // 1.6Mbps + uploadThroughput: Math.floor(750 * 1024 / 8), // 750Kbps offline: false }; From 52ef9ef59152c5c4ad4440845e5ec6c69299254f Mon Sep 17 00:00:00 2001 From: Will Chen Date: Mon, 15 May 2017 13:46:00 -0700 Subject: [PATCH 15/28] small improvements --- plots/measure.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index a401063f07bd..e8c111fe217d 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -21,8 +21,12 @@ const parseURL = require('url').parse; const mkdirp = require('mkdirp'); const args = require('yargs') + .wrap(Math.min(process.stdout.columns, 120)) .help('help') - .usage('npm run measure -- [options]') + .usage('node measure.js [options]') + .example('node $0 -n 3 --sites-path ./sample-sites.json') + .example('node $0 --site https://google.com/') + .example('node $0 --subset') .describe({ 'n': 'Number of runs to do per site', 'reuse-chrome': 'Reuse the same Chrome instance across all site runs', @@ -40,7 +44,7 @@ const args = require('yargs') .describe({ 'sites-path': 'Include relative path of a json file with urls to run', 'subset': 'Measure a subset of popular sites', - 'site': 'Include a specific site URL to run', + 'site': 'Include a specific site url to run', }) .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) .argv; @@ -221,8 +225,9 @@ function main() { console.error // eslint-disable-line no-console )); return; + } else { + runAnalysisWithNewChromeInstances(); } - runAnalysisWithNewChromeInstances(); } main(); From 72ae90d4c4c8d418b41c19d9d6191b60e1340150 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Mon, 15 May 2017 14:33:40 -0700 Subject: [PATCH 16/28] fix launcher --- plots/measure.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 56ab0e9cac3e..7b9ab1608f90 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -214,11 +214,12 @@ function main() { } if (REUSE_CHROME) { - const launcher = ChromeLauncher.launch(); - return launcher - .then(() => runAnalysisWithExistingChromeInstances()) - .catch(err => console.error(err)) - .then(() => launcher.kill()); + ChromeLauncher.launch({port: 9222}).then(launcher => { + return runAnalysisWithExistingChromeInstances() + .catch(err => console.error(err)) + .then(() => launcher.kill()); + }); + return; } else { runAnalysisWithNewChromeInstances(); } @@ -243,11 +244,12 @@ function runAnalysisWithNewChromeInstances() { const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; for (const url of URLS) { promise = promise.then(() => { - const launcher = ChromeLauncher.launch(); - return launcher - .then(() => singleRunAnalysis(url, id, {ignoreRun})) - .catch(err => console.error(err)) - .then(() => launcher.kill()); + return ChromeLauncher.launch({port: 9222}).then(launcher => { + return singleRunAnalysis(url, id, {ignoreRun}) + .catch(err => console.error(err)) + .then(() => launcher.kill()); + }) + .catch(err => console.error(err)); }); } } @@ -262,10 +264,7 @@ function runAnalysisWithNewChromeInstances() { function runAnalysisWithExistingChromeInstances() { let promise = Promise.resolve(); - // Running it n + 1 times because the first run is deliberately ignored - // because it has different perf characteristics from subsequent runs - // (e.g. DNS cache which can't be easily reset between runs) - for (let i = 0; i <= NUMBER_OF_RUNS; i++) { + for (let i = 0; i < NUMBER_OF_RUNS; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); From 963ec9ea24df3adfabc89ee384f5bfd92e16eac5 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 11:32:24 -0700 Subject: [PATCH 17/28] paul irish fb --- plots/measure.js | 73 ++++++++++++++++++++++++------------------------ plots/utils.js | 27 ------------------ 2 files changed, 37 insertions(+), 63 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 7b9ab1608f90..7981c8393784 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -59,27 +59,12 @@ const ChromeLauncher = require('../chrome-launcher/chrome-launcher.js'); const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const DISABLE_DEVICE_EMULATION = args['disable-device-emulation']; -const DISABLE_CPU_THROTTLING = args['disable-cpu-throttling']; -const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling']; -const REUSE_CHROME = args['reuse-chrome']; -const KEEP_FIRST_RUN = args['keep-first-run'] || !REUSE_CHROME; -const SITES_PATH = args['sites-path']; -const SUBSET = args['subset']; -const SITE = args['site']; -const CUSTOM_NUMBER_OF_RUNS = args['n']; +const keepFirstrun = args['keep-first-run'] || !args['reuse-chrome']; // Running it n + 1 times if the first run is deliberately ignored // because it has different perf characteristics from subsequent runs // (e.g. DNS cache which can't be easily reset between runs) -const NUMBER_OF_RUNS = (CUSTOM_NUMBER_OF_RUNS || 20) + (KEEP_FIRST_RUN ? 0 : 1); - -const FLAGS = { - output: 'json', - disableCpuThrottling: DISABLE_CPU_THROTTLING, - disableNetworkThrottling: DISABLE_NETWORK_THROTTLING, - disableDeviceEmulation: DISABLE_DEVICE_EMULATION, -}; +const numberOfRuns = (args['n'] || 3); const SITES = [ // Flagship sites @@ -168,15 +153,15 @@ const SITES = [ ]; function getUrls() { - if (SITES_PATH) { - return require(path.resolve(__dirname, SITES_PATH)); + if (args['sites-path']) { + return require(path.resolve(__dirname, args['sites-path'])); } - if (SITE) { - return [SITE]; + if (args['site']) { + return [args['site']]; } - if (SUBSET) { + if (args['subset']) { return [ 'https://en.wikipedia.org/wiki/Google', 'https://mobile.twitter.com/ChromeDevTools', @@ -207,15 +192,20 @@ function getUrls() { const URLS = getUrls(); function main() { + if (numberOfRuns === 1 && !keepFirstrun) { + console.log('ERROR: You are only doing one run and re-using chrome, but did not specify --keep-first-run'); + return; + } + if (utils.isDir(constants.OUT_PATH)) { console.log('ERROR: Found output from previous run at: ', constants.OUT_PATH); console.log('Please run: npm run clean'); return; } - if (REUSE_CHROME) { - ChromeLauncher.launch({port: 9222}).then(launcher => { - return runAnalysisWithExistingChromeInstances() + if (args['reuse-chrome']) { + ChromeLauncher.launch().then(launcher => { + return runAnalysisWithExistingChromeInstances(launcher) .catch(err => console.error(err)) .then(() => launcher.kill()); }); @@ -235,17 +225,18 @@ main(); function runAnalysisWithNewChromeInstances() { let promise = Promise.resolve(); - for (let i = 0; i < NUMBER_OF_RUNS; i++) { + for (let i = 0; i < numberOfRuns; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); const id = i.toString(); const isFirstRun = i === 0; - const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; + const ignoreRun = keepFirstrun ? false : isFirstRun; for (const url of URLS) { promise = promise.then(() => { - return ChromeLauncher.launch({port: 9222}).then(launcher => { - return singleRunAnalysis(url, id, {ignoreRun}) + return ChromeLauncher.launch().then(launcher => { + // TODO: pass in launcher + return singleRunAnalysis(url, id, launcher, {ignoreRun}) .catch(err => console.error(err)) .then(() => launcher.kill()); }) @@ -259,20 +250,21 @@ function runAnalysisWithNewChromeInstances() { /** * Reuses existing Chrome instance for all site runs. * Returns a promise chain that analyzes all the sites n times. + * @param {TODO} * @return {!Promise} */ -function runAnalysisWithExistingChromeInstances() { +function runAnalysisWithExistingChromeInstances(launcher) { let promise = Promise.resolve(); - for (let i = 0; i < NUMBER_OF_RUNS; i++) { + for (let i = 0; i < numberOfRuns; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); const id = i.toString(); const isFirstRun = i === 0; - const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun; + const ignoreRun = keepFirstrun ? false : isFirstRun; for (const url of URLS) { - promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun})); + promise = promise.then(() => singleRunAnalysis(url, id, launcher, {ignoreRun})); } } return promise; @@ -282,6 +274,7 @@ function runAnalysisWithExistingChromeInstances() { * Analyzes a site a single time using lighthouse. * @param {string} url * @param {string} id + * @param {TODO} * @param {{ignoreRun: boolean}} options * @return {!Promise} */ @@ -295,20 +288,28 @@ function singleRunAnalysis(url, id, {ignoreRun}) { } const outputPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME); const assetsPath = path.resolve(runPath, 'assets'); - return analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}); + return analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}); } /** * Runs lighthouse and save the artifacts (not used directly by plots, * but may be helpful for debugging outlier runs). + * @param {TODO} * @param {string} url * @param {string} outputPath * @param {string} assetsPath * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) { - return lighthouse(url, FLAGS, config) +function analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}) { + const flags = { + output: 'json', + disableCpuThrottling: args['disable-cpu-throttling'], + disableNetworkThrottling: args['disable-network-throttling'], + disableDeviceEmulation: args['disable-device-emulation'], + port: launcher.port, + }; + return lighthouse(url, flags, config) .then(lighthouseResults => { if (ignoreRun) { console.log('First load of site. Results not being saved to disk.'); diff --git a/plots/utils.js b/plots/utils.js index 68b450d88a5f..df9be4d6614e 100644 --- a/plots/utils.js +++ b/plots/utils.js @@ -17,7 +17,6 @@ 'use strict'; const fs = require('fs'); -const path = require('path'); /** * @param {string} path @@ -53,34 +52,8 @@ function shuffle(array) { } } -/** - * @param {string} filePath - */ -function removeRecursive(filePath) { - try { - if (fs.existsSync(filePath)) { - if (isFile(filePath)) { - fs.unlinkSync(filePath); - return; - } - const files = fs.readdirSync(filePath); - for (let i = 0; i < files.length; i++) { - const childPath = path.resolve(filePath, files[i]); - if (isDir(childPath)) - removeRecursive(childPath); - else - fs.unlinkSync(childPath); - } - fs.rmdirSync(filePath); - } - } catch (error) { - throw new Error(`Received an error: [${error}] while trying to remove: ${filePath}`); - } -} - module.exports = { isDir, isFile, shuffle, - removeRecursive, }; From 1447442a6f8f678f46dba8b95ce171044862de07 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 11:35:33 -0700 Subject: [PATCH 18/28] nit --- plots/measure.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 7981c8393784..5492b7b494fb 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -210,9 +210,8 @@ function main() { .then(() => launcher.kill()); }); return; - } else { - runAnalysisWithNewChromeInstances(); } + runAnalysisWithNewChromeInstances(); } main(); From 922d7ab998fd4452ec9fa67b961061f6c021911c Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 11:49:24 -0700 Subject: [PATCH 19/28] fixup measure --- plots/measure.js | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/plots/measure.js b/plots/measure.js index 5492b7b494fb..42eeedcb6f93 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -59,12 +59,8 @@ const ChromeLauncher = require('../chrome-launcher/chrome-launcher.js'); const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const keepFirstrun = args['keep-first-run'] || !args['reuse-chrome']; - -// Running it n + 1 times if the first run is deliberately ignored -// because it has different perf characteristics from subsequent runs -// (e.g. DNS cache which can't be easily reset between runs) -const numberOfRuns = (args['n'] || 3); +const keepFirstRun = args['keep-first-run'] || !args['reuse-chrome']; +const numberOfRuns = args['n'] || 3; const SITES = [ // Flagship sites @@ -92,7 +88,6 @@ const SITES = [ // Also removed sites that don't have significant index pages: // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", // "live.com", "adf.ly", "googleusercontent.com", - 'https://www.google.com/search?q=flowers', 'https://youtube.com', 'https://facebook.com', @@ -192,7 +187,7 @@ function getUrls() { const URLS = getUrls(); function main() { - if (numberOfRuns === 1 && !keepFirstrun) { + if (numberOfRuns === 1 && !keepFirstRun) { console.log('ERROR: You are only doing one run and re-using chrome, but did not specify --keep-first-run'); return; } @@ -230,11 +225,10 @@ function runAnalysisWithNewChromeInstances() { const id = i.toString(); const isFirstRun = i === 0; - const ignoreRun = keepFirstrun ? false : isFirstRun; + const ignoreRun = keepFirstRun ? false : isFirstRun; for (const url of URLS) { promise = promise.then(() => { return ChromeLauncher.launch().then(launcher => { - // TODO: pass in launcher return singleRunAnalysis(url, id, launcher, {ignoreRun}) .catch(err => console.error(err)) .then(() => launcher.kill()); @@ -249,7 +243,7 @@ function runAnalysisWithNewChromeInstances() { /** * Reuses existing Chrome instance for all site runs. * Returns a promise chain that analyzes all the sites n times. - * @param {TODO} + * @param {!Launcher} launcher * @return {!Promise} */ function runAnalysisWithExistingChromeInstances(launcher) { @@ -261,7 +255,7 @@ function runAnalysisWithExistingChromeInstances(launcher) { const id = i.toString(); const isFirstRun = i === 0; - const ignoreRun = keepFirstrun ? false : isFirstRun; + const ignoreRun = keepFirstRun ? false : isFirstRun; for (const url of URLS) { promise = promise.then(() => singleRunAnalysis(url, id, launcher, {ignoreRun})); } @@ -273,11 +267,11 @@ function runAnalysisWithExistingChromeInstances(launcher) { * Analyzes a site a single time using lighthouse. * @param {string} url * @param {string} id - * @param {TODO} + * @param {!Launcher} launcher * @param {{ignoreRun: boolean}} options * @return {!Promise} */ -function singleRunAnalysis(url, id, {ignoreRun}) { +function singleRunAnalysis(url, id, launcher, {ignoreRun}) { console.log('Measuring site:', url, 'run:', id); const parsedURL = parseURL(url); const urlBasedFilename = sanitizeURL(`${parsedURL.host}-${parsedURL.pathname}`); @@ -293,7 +287,7 @@ function singleRunAnalysis(url, id, {ignoreRun}) { /** * Runs lighthouse and save the artifacts (not used directly by plots, * but may be helpful for debugging outlier runs). - * @param {TODO} + * @param {!Launcher} launcher * @param {string} url * @param {string} outputPath * @param {string} assetsPath @@ -318,7 +312,7 @@ function analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun .saveAssets(lighthouseResults.artifacts, lighthouseResults.audits, assetsPath) .then(() => { lighthouseResults.artifacts = undefined; - return Printer.write(lighthouseResults, FLAGS.output, outputPath); + return Printer.write(lighthouseResults, flags.output, outputPath); }); }) .catch(err => console.error(err)); From f637160370a60d710a7f60b9c65407203808f104 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 12:01:18 -0700 Subject: [PATCH 20/28] fixup ab-screenshot --- plots/ab-screenshot/analyze.js | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index c87f681d7c68..be048cc9d088 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -22,7 +22,7 @@ const path = require('path'); const opn = require('opn'); const args = require('yargs').argv; -const RUNS = args.runs || 1; +const runs = args['runs'] || 1; const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events'); @@ -87,7 +87,7 @@ function aggregate(outPathA, outPathB) { return; } - for (let i = 0; i < RUNS; i++) { + for (let i = 0; i < runs; i++) { const runDirA = getRunDir(sitePathA, i); const runDirB = getRunDir(sitePathB, i); @@ -116,11 +116,24 @@ function aggregate(outPathA, outPathB) { /** * @param {string} sitePath * @param {number} runIndex + * @return {string} */ function getRunDir(sitePath, runIndex) { return sortAndFilterRunFolders(fs.readdirSync(sitePath))[runIndex]; } +/** + * @param {!Array} folders + * @return {!Array} + */ +function sortAndFilterRunFolders(folders) { + return folders + .filter(folder => folder !== '.DS_Store') + .map(folder => Number(folder)) + .sort((a, b) => a - b) + .map(folder => folder.toString()); +} + /** * Analyzes the screenshots for the first run of a particular site. * @param {string} runPath @@ -176,18 +189,6 @@ function analyzeSingleRunScreenshots(runPath) { } } -/** - * @param {!Array} folders - * @return {!Array} - */ -function sortAndFilterRunFolders(folders) { - return folders - .filter(folder => folder !== '.DS_Store') - .map(folder => Number(folder)) - .sort((a, b) => a - b) - .map(folder => folder.toString()); -} - /** * Marks the first screenshot that happens after a particular perf timing. * @param {SingleRunScreenshots} results From 2170a9e84531b8ceed0f1de376b4dedd02eb8e94 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 12:10:45 -0700 Subject: [PATCH 21/28] fixups --- plots/ab-screenshot/analyze.js | 4 +--- plots/measure.js | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index be048cc9d088..bf55339af322 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -129,9 +129,7 @@ function getRunDir(sitePath, runIndex) { function sortAndFilterRunFolders(folders) { return folders .filter(folder => folder !== '.DS_Store') - .map(folder => Number(folder)) - .sort((a, b) => a - b) - .map(folder => folder.toString()); + .sort((a, b) => Number(a) - Number(b)); } /** diff --git a/plots/measure.js b/plots/measure.js index 42eeedcb6f93..0a70f52abf9e 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -25,18 +25,19 @@ const mkdirp = require('mkdirp'); const args = require('yargs') .wrap(Math.min(process.stdout.columns, 120)) .help('help') - .usage('node measure.js [options]') + .usage('node $0 [options]') .example('node $0 -n 3 --sites-path ./sample-sites.json') .example('node $0 --site https://google.com/') .example('node $0 --subset') .describe({ - 'n': 'Number of runs to do per site', + 'n': 'Number of runs per site', 'reuse-chrome': 'Reuse the same Chrome instance across all site runs', - 'keep-first-run': 'By default if you use --reuse-chrome, the first run results are discarded', + 'keep-first-run': 'If you use --reuse-chrome, by default the first run results are discarded', }) .group( ['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling'], 'Chrome DevTools settings:') + .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) .describe({ 'disable-device-emulation': 'Disable Nexus 5X emulation', 'disable-cpu-throttling': 'Disable CPU throttling', @@ -48,7 +49,6 @@ const args = require('yargs') 'subset': 'Measure a subset of popular sites', 'site': 'Include a specific site url to run', }) - .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) .argv; const constants = require('./constants.js'); From 3d4c291654e5c56e1abad8f577fa8d53dd627ea0 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 12:12:53 -0700 Subject: [PATCH 22/28] make eslint happy --- plots/measure.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plots/measure.js b/plots/measure.js index 0a70f52abf9e..e6f10a48485f 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -188,7 +188,8 @@ const URLS = getUrls(); function main() { if (numberOfRuns === 1 && !keepFirstRun) { - console.log('ERROR: You are only doing one run and re-using chrome, but did not specify --keep-first-run'); + console.log('ERROR: You are only doing one run and re-using chrome'); + console.log('but did not specify --keep-first-run'); return; } From cc59d4d127aa45e2adc47e0e18463088906285c1 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 15:03:48 -0700 Subject: [PATCH 23/28] paul fb --- plots/measure.js | 131 ++++++------------------------------------ plots/sites.js | 103 +++++++++++++++++++++++++++++++++ plots/sites_subset.js | 41 +++++++++++++ 3 files changed, 160 insertions(+), 115 deletions(-) create mode 100644 plots/sites.js create mode 100644 plots/sites_subset.js diff --git a/plots/measure.js b/plots/measure.js index e6f10a48485f..b71c9b0bc0da 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -17,7 +17,7 @@ 'use strict'; /* eslint-disable no-console */ - +const fs = require('fs'); const path = require('path'); const parseURL = require('url').parse; @@ -62,126 +62,27 @@ const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); const keepFirstRun = args['keep-first-run'] || !args['reuse-chrome']; const numberOfRuns = args['n'] || 3; -const SITES = [ - // Flagship sites - 'https://nytimes.com', - 'https://flipkart.com', - 'http://www.espn.com/', - 'https://www.washingtonpost.com/pwa/', - - // TTI Tester sites - 'https://housing.com/in/buy/real-estate-hyderabad', - 'http://www.npr.org/', - 'http://www.vevo.com/', - 'https://weather.com/', - 'https://www.nasa.gov/', - 'https://vine.co/', - 'http://www.booking.com/', - 'http://www.thestar.com.my', - 'http://www.58pic.com', - 'http://www.dawn.com/', - 'https://www.ebs.in/IPS/', - - // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites - // (http://www.alexa.com/topsites) - // Removed adult websites and duplicates (e.g. google int'l websites) - // Also removed sites that don't have significant index pages: - // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", - // "live.com", "adf.ly", "googleusercontent.com", - 'https://www.google.com/search?q=flowers', - 'https://youtube.com', - 'https://facebook.com', - 'https://baidu.com', - 'https://en.wikipedia.org/wiki/Google', - 'https://yahoo.com', - 'https://amazon.com', - 'http://www.qq.com/', - 'https://taobao.com', - 'https://vk.com', - 'https://mobile.twitter.com/ChromeDevTools', - 'https://www.instagram.com/stephencurry30', - 'http://www.hao123.cn/', - 'http://www.sohu.com/', - 'https://sina.com.cn', - 'https://reddit.com', - 'https://linkedin.com', - 'https://tmall.com', - 'https://weibo.com', - 'https://360.cn', - 'https://yandex.ru', - 'https://ebay.com', - 'https://bing.com', - 'https://msn.com', - 'https://www.sogou.com/', - 'https://wordpress.com', - 'https://microsoft.com', - 'https://tumblr.com', - 'https://aliexpress.com', - 'https://blogspot.com', - 'https://netflix.com', - 'https://ok.ru', - 'https://stackoverflow.com', - 'https://imgur.com', - 'https://apple.com', - 'http://www.naver.com/', - 'https://mail.ru', - 'http://www.imdb.com/', - 'https://office.com', - 'https://github.com', - 'https://pinterest.com', - 'https://paypal.com', - 'http://www.tianya.cn/', - 'https://diply.com', - 'https://twitch.tv', - 'https://adobe.com', - 'https://wikia.com', - 'https://coccoc.com', - 'https://so.com', - 'https://fc2.com', - 'https://www.pixnet.net/', - 'https://dropbox.com', - 'https://zhihu.com', - 'https://whatsapp.com', - 'https://alibaba.com', - 'https://ask.com', - 'https://bbc.com' -]; - function getUrls() { - if (args['sites-path']) { - return require(path.resolve(__dirname, args['sites-path'])); - } - if (args['site']) { return [args['site']]; } - if (args['subset']) { - return [ - 'https://en.wikipedia.org/wiki/Google', - 'https://mobile.twitter.com/ChromeDevTools', - 'https://www.instagram.com/stephencurry30', - 'https://amazon.com', - 'https://nytimes.com', - 'https://www.google.com/search?q=flowers', - - 'https://flipkart.com', - 'http://www.espn.com/', - 'https://www.washingtonpost.com/pwa/', - 'http://www.npr.org/', - 'http://www.booking.com/', - 'https://youtube.com', - 'https://reddit.com', - 'https://ebay.com', - 'https://stackoverflow.com', - 'https://apple.com', + if (args['sites-path']) { + const sitesPath = path.resolve(__dirname, args['sites-path']); + if (path.extname(sitesPath) === '.json') { + return JSON.parse(fs.readFileSync(sitesPath, 'utf-8')); + } else if (path.extname(sitesPath) === '.js') { + return eval(fs.readFileSync(sitesPath, 'utf-8')); + } else { + throw new Error('Must pass a js or json file to --sites-path'); + } + } - // Could not run nasa on gin3g - 'https://www.nasa.gov/', - ]; + if (args['subset']) { + return eval(fs.readFileSync(path.resolve(__dirname, 'sites_subset.js'), 'utf-8')); } - return SITES; + return eval(fs.readFileSync(path.resolve(__dirname, 'sites.js'), 'utf-8')); } const URLS = getUrls(); @@ -298,8 +199,8 @@ function singleRunAnalysis(url, id, launcher, {ignoreRun}) { function analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}) { const flags = { output: 'json', - disableCpuThrottling: args['disable-cpu-throttling'], - disableNetworkThrottling: args['disable-network-throttling'], + disableCpuThrottling: ignoreRun ? true : args['disable-cpu-throttling'], + disableNetworkThrottling: ignoreRun ? true : args['disable-network-throttling'], disableDeviceEmulation: args['disable-device-emulation'], port: launcher.port, }; diff --git a/plots/sites.js b/plots/sites.js new file mode 100644 index 000000000000..c0e35e68d194 --- /dev/null +++ b/plots/sites.js @@ -0,0 +1,103 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/* eslint-disable no-unused-expressions */ +[ + // Flagship sites + 'https://nytimes.com', + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + + // TTI Tester sites + 'https://housing.com/in/buy/real-estate-hyderabad', + 'http://www.npr.org/', + 'http://www.vevo.com/', + 'https://weather.com/', + 'https://www.nasa.gov/', + 'https://vine.co/', + 'http://www.booking.com/', + 'http://www.thestar.com.my', + 'http://www.58pic.com', + 'http://www.dawn.com/', + 'https://www.ebs.in/IPS/', + + // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites + // (http://www.alexa.com/topsites) + // Removed adult websites and duplicates (e.g. google int'l websites) + // Also removed sites that don't have significant index pages: + // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com", + // "live.com", "adf.ly", "googleusercontent.com", + 'https://www.google.com/search?q=flowers', + 'https://youtube.com', + 'https://facebook.com', + 'https://baidu.com', + 'https://en.wikipedia.org/wiki/Google', + 'https://yahoo.com', + 'https://amazon.com', + 'http://www.qq.com/', + 'https://taobao.com', + 'https://vk.com', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'http://www.hao123.cn/', + 'http://www.sohu.com/', + 'https://sina.com.cn', + 'https://reddit.com', + 'https://linkedin.com', + 'https://tmall.com', + 'https://weibo.com', + 'https://360.cn', + 'https://yandex.ru', + 'https://ebay.com', + 'https://bing.com', + 'https://msn.com', + 'https://www.sogou.com/', + 'https://wordpress.com', + 'https://microsoft.com', + 'https://tumblr.com', + 'https://aliexpress.com', + 'https://blogspot.com', + 'https://netflix.com', + 'https://ok.ru', + 'https://stackoverflow.com', + 'https://imgur.com', + 'https://apple.com', + 'http://www.naver.com/', + 'https://mail.ru', + 'http://www.imdb.com/', + 'https://office.com', + 'https://github.com', + 'https://pinterest.com', + 'https://paypal.com', + 'http://www.tianya.cn/', + 'https://diply.com', + 'https://twitch.tv', + 'https://adobe.com', + 'https://wikia.com', + 'https://coccoc.com', + 'https://so.com', + 'https://fc2.com', + 'https://www.pixnet.net/', + 'https://dropbox.com', + 'https://zhihu.com', + 'https://whatsapp.com', + 'https://alibaba.com', + 'https://ask.com', + 'https://bbc.com' +]; diff --git a/plots/sites_subset.js b/plots/sites_subset.js new file mode 100644 index 000000000000..da1b7552dc8d --- /dev/null +++ b/plots/sites_subset.js @@ -0,0 +1,41 @@ +/** + * @license + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/* eslint-disable no-unused-expressions */ +[ + 'https://en.wikipedia.org/wiki/Google', + 'https://mobile.twitter.com/ChromeDevTools', + 'https://www.instagram.com/stephencurry30', + 'https://amazon.com', + 'https://nytimes.com', + 'https://www.google.com/search?q=flowers', + + 'https://flipkart.com', + 'http://www.espn.com/', + 'https://www.washingtonpost.com/pwa/', + 'http://www.npr.org/', + 'http://www.booking.com/', + 'https://youtube.com', + 'https://reddit.com', + 'https://ebay.com', + 'https://stackoverflow.com', + 'https://apple.com', + + // Could not run nasa on gin3g + 'https://www.nasa.gov/', +]; From f50a21ff429911b535ffbec849256bcb56007a41 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 16:23:56 -0700 Subject: [PATCH 24/28] paul fb --- plots/ab-screenshot/analyze.js | 2 +- plots/measure.js | 34 ++++++++++++---------------------- plots/sites.js | 3 +-- plots/sites_subset.js | 3 +-- 4 files changed, 15 insertions(+), 27 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index bf55339af322..330b946eba82 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -22,7 +22,7 @@ const path = require('path'); const opn = require('opn'); const args = require('yargs').argv; -const runs = args['runs'] || 1; +const runs = args.runs || 1; const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events'); diff --git a/plots/measure.js b/plots/measure.js index b71c9b0bc0da..32abc1955c1c 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -49,6 +49,7 @@ const args = require('yargs') 'subset': 'Measure a subset of popular sites', 'site': 'Include a specific site url to run', }) + .default('sites-path', 'sites.js') .argv; const constants = require('./constants.js'); @@ -59,30 +60,19 @@ const ChromeLauncher = require('../chrome-launcher/chrome-launcher.js'); const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); -const keepFirstRun = args['keep-first-run'] || !args['reuse-chrome']; -const numberOfRuns = args['n'] || 3; +const keepFirstRun = args.keepFirstRun || !args.reuseChrome; +const numberOfRuns = args.n || 3; function getUrls() { - if (args['site']) { - return [args['site']]; + if (args.site) { + return [args.site]; } - if (args['sites-path']) { - const sitesPath = path.resolve(__dirname, args['sites-path']); - if (path.extname(sitesPath) === '.json') { - return JSON.parse(fs.readFileSync(sitesPath, 'utf-8')); - } else if (path.extname(sitesPath) === '.js') { - return eval(fs.readFileSync(sitesPath, 'utf-8')); - } else { - throw new Error('Must pass a js or json file to --sites-path'); - } - } - - if (args['subset']) { - return eval(fs.readFileSync(path.resolve(__dirname, 'sites_subset.js'), 'utf-8')); + if (args.subset) { + return require(path.resolve(__dirname, 'sites_subset.js')); } - return eval(fs.readFileSync(path.resolve(__dirname, 'sites.js'), 'utf-8')); + return require(path.resolve(__dirname, args.sitesPath)); } const URLS = getUrls(); @@ -100,7 +90,7 @@ function main() { return; } - if (args['reuse-chrome']) { + if (args.reuseChrome) { ChromeLauncher.launch().then(launcher => { return runAnalysisWithExistingChromeInstances(launcher) .catch(err => console.error(err)) @@ -199,9 +189,9 @@ function singleRunAnalysis(url, id, launcher, {ignoreRun}) { function analyzeWithLighthouse(launcher, url, outputPath, assetsPath, {ignoreRun}) { const flags = { output: 'json', - disableCpuThrottling: ignoreRun ? true : args['disable-cpu-throttling'], - disableNetworkThrottling: ignoreRun ? true : args['disable-network-throttling'], - disableDeviceEmulation: args['disable-device-emulation'], + disableCpuThrottling: ignoreRun ? true : args.disableCpuThrottling, + disableNetworkThrottling: ignoreRun ? true : args.disableNetworkThrottling, + disableDeviceEmulation: args.disableDeviceEmulation, port: launcher.port, }; return lighthouse(url, flags, config) diff --git a/plots/sites.js b/plots/sites.js index c0e35e68d194..2930d3a86fe7 100644 --- a/plots/sites.js +++ b/plots/sites.js @@ -16,8 +16,7 @@ */ 'use strict'; -/* eslint-disable no-unused-expressions */ -[ +module.exports = [ // Flagship sites 'https://nytimes.com', 'https://flipkart.com', diff --git a/plots/sites_subset.js b/plots/sites_subset.js index da1b7552dc8d..a8bcbe24be2b 100644 --- a/plots/sites_subset.js +++ b/plots/sites_subset.js @@ -16,8 +16,7 @@ */ 'use strict'; -/* eslint-disable no-unused-expressions */ -[ +module.exports = [ 'https://en.wikipedia.org/wiki/Google', 'https://mobile.twitter.com/ChromeDevTools', 'https://www.instagram.com/stephencurry30', From 85551292dc6e338a50466973c1a74e2a69727ca5 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 16:34:25 -0700 Subject: [PATCH 25/28] fixup --- plots/ab-screenshot/analyze.js | 8 ++++---- plots/measure.js | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plots/ab-screenshot/analyze.js b/plots/ab-screenshot/analyze.js index 330b946eba82..af0f90cf0f0c 100644 --- a/plots/ab-screenshot/analyze.js +++ b/plots/ab-screenshot/analyze.js @@ -20,9 +20,9 @@ const fs = require('fs'); const path = require('path'); const opn = require('opn'); -const args = require('yargs').argv; - -const runs = args.runs || 1; +const args = require('yargs') + .default('runs', 1) + .argv; const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events'); @@ -87,7 +87,7 @@ function aggregate(outPathA, outPathB) { return; } - for (let i = 0; i < runs; i++) { + for (let i = 0; i < args.runs; i++) { const runDirA = getRunDir(sitePathA, i); const runDirB = getRunDir(sitePathB, i); diff --git a/plots/measure.js b/plots/measure.js index 32abc1955c1c..d99d1f4e4b55 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -17,7 +17,6 @@ 'use strict'; /* eslint-disable no-console */ -const fs = require('fs'); const path = require('path'); const parseURL = require('url').parse; From e93c30011b19c1131ce5c2068ae996808221dad1 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 17:32:36 -0700 Subject: [PATCH 26/28] paul fb --- plots/README.md | 10 +++------- plots/{plot-per-site.html => bars-per-site.html} | 2 +- plots/{plot-per-site.js => bars-per-site.js} | 0 plots/measure.js | 10 +++++----- 4 files changed, 9 insertions(+), 13 deletions(-) rename plots/{plot-per-site.html => bars-per-site.html} (94%) rename plots/{plot-per-site.js => bars-per-site.js} (100%) diff --git a/plots/README.md b/plots/README.md index 07f45b623980..50809584f5b4 100644 --- a/plots/README.md +++ b/plots/README.md @@ -14,17 +14,13 @@ You need to build lighthouse first. ### Generating & viewing charts ``` -# View all commands -$ cd plots -$ yarn run - # Run lighthouse to collect metrics data -$ yarn measure +$ node measure.js # Analyze the data to generate a summary file (i.e. out/generatedResults.js) # This will launch the charts web page in the browser -$ yarn analyze +$ node analyze.js # If you need to view the charts later -$ yarn open +$ node open.js ``` diff --git a/plots/plot-per-site.html b/plots/bars-per-site.html similarity index 94% rename from plots/plot-per-site.html rename to plots/bars-per-site.html index ee68574ae156..289948f5d222 100644 --- a/plots/plot-per-site.html +++ b/plots/bars-per-site.html @@ -23,7 +23,7 @@ - + diff --git a/plots/plot-per-site.js b/plots/bars-per-site.js similarity index 100% rename from plots/plot-per-site.js rename to plots/bars-per-site.js diff --git a/plots/measure.js b/plots/measure.js index d99d1f4e4b55..1407c07421a5 100644 --- a/plots/measure.js +++ b/plots/measure.js @@ -33,9 +33,10 @@ const args = require('yargs') 'reuse-chrome': 'Reuse the same Chrome instance across all site runs', 'keep-first-run': 'If you use --reuse-chrome, by default the first run results are discarded', }) + .default('n', 3) .group( ['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling'], - 'Chrome DevTools settings:') + 'Lighthouse settings:') .boolean(['disable-device-emulation', 'disable-cpu-throttling', 'disable-network-throttling']) .describe({ 'disable-device-emulation': 'Disable Nexus 5X emulation', @@ -60,7 +61,6 @@ const Printer = require('../lighthouse-cli/printer'); const assetSaver = require('../lighthouse-core/lib/asset-saver.js'); const keepFirstRun = args.keepFirstRun || !args.reuseChrome; -const numberOfRuns = args.n || 3; function getUrls() { if (args.site) { @@ -77,7 +77,7 @@ function getUrls() { const URLS = getUrls(); function main() { - if (numberOfRuns === 1 && !keepFirstRun) { + if (args.n === 1 && !keepFirstRun) { console.log('ERROR: You are only doing one run and re-using chrome'); console.log('but did not specify --keep-first-run'); return; @@ -110,7 +110,7 @@ main(); function runAnalysisWithNewChromeInstances() { let promise = Promise.resolve(); - for (let i = 0; i < numberOfRuns; i++) { + for (let i = 0; i < args.n; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); @@ -140,7 +140,7 @@ function runAnalysisWithNewChromeInstances() { function runAnalysisWithExistingChromeInstances(launcher) { let promise = Promise.resolve(); - for (let i = 0; i < numberOfRuns; i++) { + for (let i = 0; i < args.n; i++) { // Averages out any order-dependent effects such as memory pressure utils.shuffle(URLS); From 92a927ab4b8b041e22cdb3200913160b35687249 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Tue, 30 May 2017 17:46:33 -0700 Subject: [PATCH 27/28] add links --- plots/bars-per-site.html | 10 ++++++++++ plots/index.html | 1 + plots/metrics-per-site.html | 1 + 3 files changed, 12 insertions(+) diff --git a/plots/bars-per-site.html b/plots/bars-per-site.html index 289948f5d222..880e5ca612e8 100644 --- a/plots/bars-per-site.html +++ b/plots/bars-per-site.html @@ -19,9 +19,19 @@ + + diff --git a/plots/index.html b/plots/index.html index ea0f99f7ff61..cd7a5ca67fa6 100644 --- a/plots/index.html +++ b/plots/index.html @@ -31,6 +31,7 @@ View results: grouped by metric grouped by site + bar charts per site diff --git a/plots/metrics-per-site.html b/plots/metrics-per-site.html index 0daf43afae1c..a82146bcc073 100644 --- a/plots/metrics-per-site.html +++ b/plots/metrics-per-site.html @@ -30,6 +30,7 @@ View results: grouped by metric grouped by site + bar charts per site From ebdea6ea46b601827b652ad9ffff5a40f3a0b02e Mon Sep 17 00:00:00 2001 From: Will Chen Date: Wed, 31 May 2017 10:28:37 -0700 Subject: [PATCH 28/28] spruce up readme --- plots/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/plots/README.md b/plots/README.md index 50809584f5b4..5f3399d2fdf7 100644 --- a/plots/README.md +++ b/plots/README.md @@ -24,3 +24,33 @@ $ node analyze.js # If you need to view the charts later $ node open.js ``` + +### Advanced usage + +``` +$ node measure.js --help + +node measure.js [options] + +Lighthouse settings: + --disable-device-emulation Disable Nexus 5X emulation [boolean] + --disable-cpu-throttling Disable CPU throttling [boolean] + --disable-network-throttling Disable network throttling [boolean] + +Options to specify sites: + --sites-path Include relative path of a json file with urls to run [default: "sites.js"] + --subset Measure a subset of popular sites + --site Include a specific site url to run + +Options: + --help Show help [boolean] + -n Number of runs per site [default: 3] + --reuse-chrome Reuse the same Chrome instance across all site runs + --keep-first-run If you use --reuse-chrome, by default the first run results are discarded + +Examples: + node measure.js -n 3 --sites-path ./sample-sites.json + node measure.js --site https://google.com/ + node measure.js --subset + +``` \ No newline at end of file