Plots: make measure script more flexible (CLI args)

.gitignore

@@ -42,4 +42,4 @@ lighthouse-cli/types/*.map
 lighthouse-core/report/partials/templates/
 lighthouse-core/report/templates/*.js
 
-plots/out/
+plots/out**

lighthouse-core/lib/emulation.js

@@ -41,10 +41,13 @@ const NEXUS5X_USERAGENT = {
     '(KHTML, like Gecko) Chrome/59.0.3033.0 Mobile Safari/537.36'
 };
 
+const LATENCY_FACTOR = 3.75;
+const THROUGHPUT_FACTOR = 1.1;
+
 const TYPICAL_MOBILE_THROTTLING_METRICS = {
-  latency: 150, // 150ms
-  downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8), // 1.6Mbps
-  uploadThroughput: Math.floor(750 * 1024 / 8), // 750Kbps
+  latency: 150 * LATENCY_FACTOR, // 150ms
+  downloadThroughput: Math.floor(1.6 * 1024 * 1024 / 8) / THROUGHPUT_FACTOR, // 1.6Mbps
+  uploadThroughput: Math.floor(750 * 1024 / 8) / THROUGHPUT_FACTOR, // 750Kbps
   offline: false
 };
 

plots/ab-screenshot/analyze.js

@@ -22,6 +22,8 @@ const path = require('path');
 const opn = require('opn');
 const args = require('yargs').argv;
 
+const RUNS = args.runs || 1;
+
 const Metrics = require('../../lighthouse-core/lib/traces/pwmetrics-events');
 
 const constants = require('../constants');
@@ -84,25 +86,47 @@ function aggregate(outPathA, outPathB) {
     if (!utils.isDir(sitePathB)) {
       return;
     }
-    const siteScreenshotsComparison = {
-      siteName: siteDir,
-      runA: analyzeSingleRunScreenshots(sitePathA),
-      runB: analyzeSingleRunScreenshots(sitePathB)
-    };
-    results.push(siteScreenshotsComparison);
+
+    for (let i = 0; i < RUNS; i++) {
+      const runDirA = getRunDir(sitePathA, i);
+      const runDirB = getRunDir(sitePathB, i);
+
+      const runPathA = path.resolve(sitePathA, runDirA);
+      const runPathB = path.resolve(sitePathB, runDirB);
+
+      const lighthouseFileA = path.resolve(runPathA, constants.LIGHTHOUSE_RESULTS_FILENAME);
+      const lighthouseFileB = path.resolve(runPathB, constants.LIGHTHOUSE_RESULTS_FILENAME);
+
+      if (!utils.isFile(lighthouseFileA) || !utils.isFile(lighthouseFileB)) {
+        continue;
+      }
+
+      const siteScreenshotsComparison = {
+        siteName: `${siteDir} runA: ${runDirA} runB: ${runDirB}`,
+        runA: analyzeSingleRunScreenshots(runPathA),
+        runB: analyzeSingleRunScreenshots(runPathB),
+      };
+      results.push(siteScreenshotsComparison);
+    }
   });
 
   return results;
 }
 
 /**
- * Analyzes the screenshots for the first run of a particular site.
  * @param {string} sitePath
+ * @param {number} runIndex
+ */
+function getRunDir(sitePath, runIndex) {
+  return sortAndFilterRunFolders(fs.readdirSync(sitePath))[runIndex];
+}
+
+/**
+ * Analyzes the screenshots for the first run of a particular site.
+ * @param {string} runPath
  * @return {!SingleRunScreenshots}
  */
-function analyzeSingleRunScreenshots(sitePath) {
-  const runDir = sortAndFilterRunFolders(fs.readdirSync(sitePath))[0];
-  const runPath = path.resolve(sitePath, runDir);
+function analyzeSingleRunScreenshots(runPath) {
   const lighthouseResultsPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME);
   const lighthouseResults = JSON.parse(fs.readFileSync(lighthouseResultsPath));
 

plots/measure.js

@@ -20,6 +20,7 @@ const path = require('path');
 const parseURL = require('url').parse;
 
 const mkdirp = require('mkdirp');
+const args = require('yargs').argv;
 
 const constants = require('./constants.js');
 const utils = require('./utils.js');
@@ -29,9 +30,47 @@ const ChromeLauncher = require('../lighthouse-cli/chrome-launcher.js').ChromeLau
 const Printer = require('../lighthouse-cli/printer');
 const assetSaver = require('../lighthouse-core/lib/asset-saver.js');
 
-const NUMBER_OF_RUNS = 20;
+const DISABLE_DEVICE_EMULATION = false;
+const DISABLE_CPU_THROTTLING = true;
+const DISABLE_NETWORK_THROTTLING = args['disable-network-throttling'] || false;
+const KEEP_FIRST_RUN = args['keep-first-run'];
 
-const URLS = [
+// Running it n + 1 times if the first run is deliberately ignored
+// because it has different perf characteristics from subsequent runs
+// (e.g. DNS cache which can't be easily reset between runs)
+const NUMBER_OF_RUNS = (args.n || 20) + (KEEP_FIRST_RUN ? 0 : 1);
+
+const FLAGS = {
+  output: 'json',
+  disableCpuThrottling: DISABLE_CPU_THROTTLING,
+  disableNetworkThrottling: DISABLE_NETWORK_THROTTLING,
+  disableDeviceEmulation: DISABLE_DEVICE_EMULATION,
+};
+console.log('Running lighthouse with flag\n disableNetworkThrottling: ', FLAGS.disableNetworkThrottling);
+
+const SUBSET = [
+  'https://en.wikipedia.org/wiki/Google',
+  'https://mobile.twitter.com/ChromeDevTools',
+  'https://www.instagram.com/stephencurry30',
+  'https://amazon.com',
+  'https://nytimes.com',
+  'https://www.google.com/search?q=flowers',
+
+  'https://flipkart.com',
+  'http://www.espn.com/',
+  'https://www.washingtonpost.com/pwa/',
+  'http://www.npr.org/',
+  'http://www.booking.com/',
+  'https://youtube.com',
+  'https://reddit.com',
+  'https://ebay.com',
+  'https://stackoverflow.com',
+  'https://apple.com',
+
+  // Could not run nasa on gin3g
+  // 'https://www.nasa.gov/',
+];
+const URLS = args.subset ? SUBSET : args.site ? [args.site] : [
   // Flagship sites
   'https://nytimes.com',
   'https://flipkart.com',
@@ -51,25 +90,25 @@ const URLS = [
   'http://www.dawn.com/',
   'https://www.ebs.in/IPS/',
 
-  // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites
-  // (http://www.alexa.com/topsites)
-  // Removed adult websites and duplicates (e.g. google int'l websites)
-  // Also removed sites that don't have significant index pages:
-  // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com",
-  // "live.com", "adf.ly", "googleusercontent.com",
+  // // Sourced from: https://en.wikipedia.org/wiki/List_of_most_popular_websites
+  // // (http://www.alexa.com/topsites)
+  // // Removed adult websites and duplicates (e.g. google int'l websites)
+  // // Also removed sites that don't have significant index pages:
+  // // "t.co", "popads.net", "onclickads.net", "microsoftonline.com", "onclckds.com", "cnzz.com",
+  // // "live.com", "adf.ly", "googleusercontent.com",
 
-  'https://google.com',
+  'https://www.google.com/search?q=flowers',
   'https://youtube.com',
   'https://facebook.com',
   'https://baidu.com',
-  'https://wikipedia.org',
+  'https://en.wikipedia.org/wiki/Google',
   'https://yahoo.com',
   'https://amazon.com',
   'http://www.qq.com/',
   'https://taobao.com',
   'https://vk.com',
-  'https://twitter.com',
-  'https://instagram.com',
+  'https://mobile.twitter.com/ChromeDevTools',
+  'https://www.instagram.com/stephencurry30',
   'http://www.hao123.cn/',
   'http://www.sohu.com/',
   'https://sina.com.cn',
@@ -130,18 +169,7 @@ function main() {
     return;
   }
 
-  const launcher = new ChromeLauncher();
-  launcher
-    .isDebuggerReady()
-    .catch(() => launcher.run())
-    .then(() => runAnalysis())
-    .then(() => launcher.kill())
-    .catch(err => launcher.kill().then(
-      () => {
-        throw err;
-      },
-      console.error // eslint-disable-line no-console
-    ));
+  runAnalysisWithNewChromeInstances()
 }
 
 main();
@@ -150,20 +178,30 @@ main();
  * Returns a promise chain that analyzes all the sites n times.
  * @return {!Promise}
  */
-function runAnalysis() {
+function runAnalysisWithNewChromeInstances() {
   let promise = Promise.resolve();
 
-  // Running it n + 1 times because the first run is deliberately ignored
-  // because it has different perf characteristics from subsequent runs
-  // (e.g. DNS cache which can't be easily reset between runs)
-  for (let i = 0; i <= NUMBER_OF_RUNS; i++) {
+  for (let i = 0; i < NUMBER_OF_RUNS; i++) {
     // Averages out any order-dependent effects such as memory pressure
     utils.shuffle(URLS);
 
     const id = i.toString();
     const isFirstRun = i === 0;
+    const ignoreRun = KEEP_FIRST_RUN ? false : isFirstRun;
     for (const url of URLS) {
-      promise = promise.then(() => singleRunAnalysis(url, id, {ignoreRun: isFirstRun}));
+      promise = promise.then(() => {
+        const launcher = new ChromeLauncher();
+        return launcher.isDebuggerReady()
+          .catch(() => launcher.run())
+          .then(() => singleRunAnalysis(url, id, { ignoreRun }))
+          .then(() => launcher.kill())
+          .catch(err => launcher.kill().then(
+            () => {
+              throw err;
+            },
+            console.error // eslint-disable-line no-console
+          ));
+      });
     }
   }
   return promise;
@@ -176,7 +214,7 @@ function runAnalysis() {
  * @param {{ignoreRun: boolean}} options
  * @return {!Promise}
  */
-function singleRunAnalysis(url, id, {ignoreRun}) {
+function singleRunAnalysis(url, id, { ignoreRun }) {
   console.log('Measuring site:', url, 'run:', id); // eslint-disable-line no-console
   const parsedURL = parseURL(url);
   const urlBasedFilename = sanitizeURL(`${parsedURL.host}-${parsedURL.pathname}`);
@@ -186,7 +224,7 @@ function singleRunAnalysis(url, id, {ignoreRun}) {
   }
   const outputPath = path.resolve(runPath, constants.LIGHTHOUSE_RESULTS_FILENAME);
   const assetsPath = path.resolve(runPath, 'assets');
-  return analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun});
+  return analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun });
 }
 
 /**
@@ -198,9 +236,8 @@ function singleRunAnalysis(url, id, {ignoreRun}) {
  * @param {{ignoreRun: boolean}} options
  * @return {!Promise}
  */
-function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) {
-  const flags = {output: 'json'};
-  return lighthouse(url, flags, config)
+function analyzeWithLighthouse(url, outputPath, assetsPath, { ignoreRun }) {
+  return lighthouse(url, FLAGS, config)
     .then(lighthouseResults => {
       if (ignoreRun) {
         return;
@@ -209,7 +246,7 @@ function analyzeWithLighthouse(url, outputPath, assetsPath, {ignoreRun}) {
         .saveAssets(lighthouseResults.artifacts, lighthouseResults.audits, assetsPath)
         .then(() => {
           lighthouseResults.artifacts = undefined;
-          return Printer.write(lighthouseResults, flags.output, outputPath);
+          return Printer.write(lighthouseResults, FLAGS.output, outputPath);
         });
     })
     .catch(err => console.error(err)); // eslint-disable-line no-console

plots/plot-per-site.html

@@ -0,0 +1,29 @@
+<!--
+Copyright 2017 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<!doctype html>
+<html>
+
+<head>
+  <script src="https://cdn.plot.ly/plotly-1.25.2.min.js"></script>
+</head>
+
+<body>
+  <script src="./out/generatedResults.js"></script>
+  <script src="./plot-per-site.js"></script>
+</body>
+
+</html>

plots/plot-per-site.js

@@ -0,0 +1,86 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+/* global Plotly, generatedResults */
+/* eslint-env browser */
+
+const IGNORED_METRICS = new Set(['Navigation Start']);
+
+const metrics = Object.keys(generatedResults).filter(metric => !IGNORED_METRICS.has(metric));
+
+let elementId = 1;
+
+/**
+ * Incrementally renders the plot, otherwise it hangs the browser
+ * because it's generating so many charts.
+ */
+const queuedPlots = [];
+
+function enqueuePlot(fn) {
+  const isFirst = queuedPlots.length == 0;
+  queuedPlots.push(fn);
+  if (isFirst) {
+    renderPlots();
+  }
+}
+
+function renderPlots() {
+  window.requestAnimationFrame(_ => {
+    const plotFn = queuedPlots.shift();
+    if (plotFn) {
+      plotFn();
+      renderPlots();
+    }
+  });
+}
+
+function createChartElement(height = 800) {
+  const div = document.createElement('div');
+  div.style = `width: 100%; height: ${height}px`;
+  div.id = 'chart' + elementId++;
+  document.body.appendChild(div);
+  return div.id;
+}
+
+function generateGroupedBarChart() {
+  const sitesCount = metrics.reduce(
+    (acc, metric) => Math.max(acc, generatedResults[metric].length),
+    0
+  );
+  for (let i = 0; i < sitesCount; i++) {
+    const data = metrics.map(metric => ({
+      y: generatedResults[metric][i].metrics.map(m => m ? m.timing : null),
+      name: metric,
+      type: 'bar'
+    }));
+
+    const layout = {
+      yaxis: {
+        rangemode: 'tozero'
+      },
+      hovermode: 'closest',
+      barmode: 'group',
+      title: generatedResults[metrics[0]][i].site
+    };
+    enqueuePlot(_ => {
+      Plotly.newPlot(createChartElement(), data, layout);
+    });
+  }
+}
+
+generateGroupedBarChart();