core(lib): add error reporting (#2420)

* refactor(StartUrl): switch from error to debugString object

* feat: add error reporting

* cave to peer pressure

* lint

* merge fixes

* reduce yarn churn

* limit impact to run.ts

* clang format

* avoid duplicate errors

* lint

* disable error reporting on test

* feedback

* remove enableErrorReporting default

* update readme

* fix yarn.lock

* add error reporting doc

* doc feedback

* move sentry call

* more cleanup

* remove unused environmentData

CONTRIBUTING.md

@@ -64,6 +64,42 @@ Don't:
 If no reference doc exists yet, then you can use the `helpText` as a stopgap for explaining
 both why the audit is important and how to fix it.
 
+## Tracking Errors
+
+We track our errors in the wild with Sentry. In general, do not worry about wrapping your audits or gatherers in try/catch blocks and reporting every error that could possibly occur; `lighthouse-core/runner.js` and `lighthouse-core/gather/gather-runner.js` already catch and report any errors that occur while running a gatherer or audit, including errors fatal to the entire run. However, there are some situations when you might want to expliticly handle an error and report it to Sentry or wrap it to avoid reporting. Generally, you can interact with Sentry simply by requiring the `lighthouse-core/lib/sentry.js` file and call its methods. The module exports a delegate that will correctly handle the error reporting based on the user's opt-in preference and will simply no-op if they haven't so you don't need to check.
+
+
+#### If you have an expected error that is recoverable but want to track how frequently it happens, *use Sentry.captureException*.
+
+```js
+const Sentry = require('./lighthouse-core/lib/sentry');
+
+try {
+  doRiskyThing();
+} catch (err) {
+  Sentry.captureException(err, {
+    tags: {audit: 'audit-name'},
+    level: 'warning',
+  });
+  doFallbackThing();
+}
+```
+
+#### If you need to track a code path that doesn't necessarily mean an error occurred, *use Sentry.captureMessage*.
+
+NOTE: If the message you're capturing is dynamic/based on user data or you need a stack trace, then create a fake error instead and use `Sentry.captureException` so that the instances will be grouped together in Sentry.
+
+```js
+const Sentry = require('./lighthouse-core/lib/sentry');
+
+if (networkRecords.length === 1) {
+  Sentry.captureMessage('Site only had 1 network request');
+  return null;
+} else {
+  // do your thang
+}
+```
+
 # For Maintainers
 
 ## Release guide

docs/error-reporting.md

@@ -0,0 +1,31 @@
+# Error Reporting Explained
+
+## What's going on?
+
+The Lighthouse team is constantly trying to improve the reliability of our tools, so we've added error tracking functionality to the CLI. Given your consent, we would like to anonymously report runtime exceptions using [Sentry](https://sentry.io/welcome/). We will use this information to detect new bugs and avoid regressions.
+
+Only CLI users are currently impacted. DevTools, extension, and node module users will not have errors reported.
+
+## What will happen if I opt-in?
+Runtime exceptions will be reported to the team along with information on your environment such as the URL you tested, your OS, and Chrome version. See [what data gets reported](#what-data-gets-reported).
+
+## What will happen if I do not opt-in?
+Runtime exceptions will not be reported to the team. Your ability to use Lighthouse will not be affected in any way.
+
+## What data gets reported?
+
+* The URL you tested
+* The runtime settings used (throttling enabled/disabled, emulation, etc)
+* The message, stack trace, and associated data of the error
+* The file path of Lighthouse node module on your machine
+* Your Lighthouse version
+* Your Chrome version
+* Your operating system
+
+## How do I opt-in?
+The first time you run the CLI you will be prompted with a message asking you if Lighthouse can anonymously report runtime exceptions. You can give a direct response of `yes` or `no` (`y`, `n`, and pressing enter which defaults to `no` are also acceptable responses), and you will not be prompted again. If no response is given within 20 seconds, a `no` response will be assumed, and you will not be prompted again. Non-interactive terminal sessions and invocations with the `CI` environment variable set will automatically not be prompted and will not opt-in by default.
+
+The CLI also has two flags to control error reporting that will override the saved preference. Running Lighthouse with `--enable-error-reporting` will report errors regardless of the saved preference, and running Lighthouse with `--no-enable-error-reporting` will *not* report errors regardless of the saved preferences.
+
+## How do I change my opt-in preference?
+Your response to the prompt will be saved to your home directory `~/.config/configstore/lighthouse.json` and used on future runs. To trigger a re-prompt, simply delete this file and Lighthouse will ask again on the next run. You can also edit this json file directly or run Lighthouse with the `--[no-]enable-error-reporting` flags.

lighthouse-cli/bin.ts

@@ -5,6 +5,7 @@
  */
 'use strict';
 
+import {existsSync} from 'fs';
 import * as path from 'path';
 
 import * as Commands from './commands/commands';
@@ -15,10 +16,15 @@ import {runLighthouse} from './run';
 const log = require('lighthouse-logger');
 const perfOnlyConfig = require('../lighthouse-core/config/perf.json');
 const pkg = require('../package.json');
+const Sentry = require('../lighthouse-core/lib/sentry');
 
 // accept noop modules for these, so the real dependency is optional.
 import {updateNotifier} from './shim-modules';
+import {askPermission} from './sentry-prompt';
 
+function isDev() {
+  return existsSync(path.join(__dirname, '../.git'));
+}
 
 // Tell user if there's a newer version of LH.
 updateNotifier({pkg}).notify();
@@ -59,6 +65,23 @@ if (cliFlags.output === Printer.OutputMode[Printer.OutputMode.json] && !cliFlags
   cliFlags.outputPath = 'stdout';
 }
 
-export function run() {
+export async function run() {
+  if (typeof cliFlags.enableErrorReporting === 'undefined') {
+    cliFlags.enableErrorReporting = await askPermission();
+  }
+
+  Sentry.init({
+    url,
+    flags: cliFlags,
+    environmentData: {
+      name: 'redacted', // prevent sentry from using hostname
+      environment: isDev() ? 'development' : 'production',
+      release: pkg.version,
+      tags: {
+        channel: 'cli',
+      },
+    },
+  });
+
   return runLighthouse(url, cliFlags, config);
 }

lighthouse-cli/cli-flags.ts

@@ -14,7 +14,7 @@ import {GetValidOutputOptions, OutputMode} from './printer';
 export interface Flags {
   port: number, chromeFlags: string, output: any, outputPath: string, saveArtifacts: boolean,
       saveAssets: boolean, view: boolean, maxWaitForLoad: number, logLevel: string,
-      hostname: string, blockedUrlPatterns: string[]
+      hostname: string, blockedUrlPatterns: string[], enableErrorReporting: boolean
 }
 
 export function getFlags(manualArgv?: string) {
@@ -54,10 +54,12 @@ export function getFlags(manualArgv?: string) {
           [
             'save-assets', 'save-artifacts', 'list-all-audits', 'list-trace-categories',
             'additional-trace-categories', 'config-path', 'chrome-flags', 'perf', 'port',
-            'hostname', 'max-wait-for-load'
+            'hostname', 'max-wait-for-load', 'enable-error-reporting'
           ],
           'Configuration:')
       .describe({
+        'enable-error-reporting':
+            'Enables error reporting (prompts once by default, setting this flag will force error reporting to that state).',
         'blocked-url-patterns': 'Block any network requests to the specified URL patterns',
         'disable-storage-reset':
             'Disable clearing the browser cache and other storage APIs before a run',

lighthouse-cli/sentry-prompt.ts

@@ -0,0 +1,63 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+import {Configstore, inquirer} from './shim-modules';
+
+const log = require('lighthouse-logger');
+
+const MAXIMUM_WAIT_TIME = 20 * 1000;
+// clang-format off
+const MESSAGE =
+  `${log.reset}We're constantly trying to improve Lighthouse and its reliability.\n  ` +
+  `May we anonymously report runtime exceptions to improve the tool over time?\n  ` +
+  `${log.reset}Learn more: https://github.com/GoogleChrome/lighthouse/blob/master/docs/error-reporting.md`;
+// clang-format on
+
+async function prompt() {
+  if (!process.stdout.isTTY || process.env.CI) {
+    // Default non-interactive sessions to false
+    return false;
+  }
+
+  let timeout: NodeJS.Timer;
+
+  const prompt = inquirer.prompt([
+    {
+      type: 'confirm',
+      name: 'isErrorReportingEnabled',
+      default: false,
+      message: MESSAGE,
+    },
+  ]);
+
+  const timeoutPromise = new Promise((resolve: (a: boolean) => {}) => {
+    timeout = setTimeout(() => {
+      prompt.ui.close();
+      process.stdout.write('\n');
+      log.warn('CLI', 'No response to error logging preference, errors will not be reported.');
+      resolve(false);
+    }, MAXIMUM_WAIT_TIME);
+  });
+
+  return Promise.race([
+    prompt.then((result: {isErrorReportingEnabled: boolean}) => {
+      clearTimeout(timeout);
+      return result.isErrorReportingEnabled;
+    }),
+    timeoutPromise,
+  ]);
+}
+
+export async function askPermission() {
+  const configstore = new Configstore('lighthouse');
+  let isErrorReportingEnabled = configstore.get('isErrorReportingEnabled');
+  if (typeof isErrorReportingEnabled === 'boolean') {
+    return isErrorReportingEnabled;
+  }
+
+  isErrorReportingEnabled = await prompt();
+  configstore.set('isErrorReportingEnabled', isErrorReportingEnabled);
+  return isErrorReportingEnabled;
+}

lighthouse-cli/shim-modules.ts

@@ -30,4 +30,27 @@ try {
   };
 }
 
-export {opn, updateNotifier};
+let inquirer;
+try {
+  inquirer = require('inquirer');
+} catch (e) {
+  inquirer = {
+    prompt() {
+      console.error('module `inquirer` not installed. Not reporting errors.');
+      return Promise.resolve({isErrorReportingEnabled: false});
+    }
+  }
+}
+
+let Configstore;
+try {
+  Configstore = require('configstore');
+} catch (e) {
+  Configstore = function Configstore() {
+    console.error('module `configstore` not installed. Not persisting user choices.');
+    this.get = () => false;
+    this.set = () => undefined;
+  }
+}
+
+export {opn, updateNotifier, inquirer, Configstore};

lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js

@@ -6,6 +6,7 @@
 'use strict';
 
 const Audit = require('../audit');
+const Sentry = require('../../lib/sentry');
 const Util = require('../../report/v2/renderer/util');
 
 const KB_IN_BYTES = 1024;
@@ -134,6 +135,15 @@ class UnusedBytes extends Audit {
 
     const tableDetails = Audit.makeTableDetails(result.headings, results);
 
+    if (debugString) {
+      // Track these with Sentry since we don't want to fail the entire audit for a single image failure.
+      // Use captureException to preserve the stack and take advantage of Sentry grouping.
+      Sentry.captureException(new Error(debugString), {
+        tags: {audit: this.meta.name},
+        level: 'warning',
+      });
+    }
+
     return {
       debugString,
       displayValue,

lighthouse-core/audits/load-fast-enough-for-pwa.js

@@ -14,6 +14,7 @@
 const Audit = require('./audit');
 const URL = require('../lib/url-shim');
 const Emulation = require('../lib/emulation');
+const Sentry = require('../lib/sentry');
 const Util = require('../report/v2/renderer/util.js');
 
 // Maximum TTFI to be considered "fast" for PWA baseline checklist
@@ -110,6 +111,22 @@ class LoadFastEnough4Pwa extends Audit {
         }
 
         if (!areLatenciesAll3G) {
+          const sentryContext = Sentry.getContext();
+          const hadThrottlingEnabled = sentryContext && sentryContext.extra &&
+              sentryContext.extra.networkThrottling;
+
+          if (hadThrottlingEnabled) {
+            // Track these instances in Sentry since there should be no requests that are fast when
+            // throttling is enabled, and it's likely a throttling bug we should look into.
+            const violatingLatency = firstRequestLatencies
+                .find(item => Number(item.latency) < latency3gMin);
+            Sentry.captureMessage('Network request latencies were not realistic', {
+              tags: {audit: this.meta.name},
+              extra: {violatingLatency},
+              level: 'warning',
+            });
+          }
+
           return {
             rawValue: true,
             // eslint-disable-next-line max-len

lighthouse-core/gather/computed/trace-of-tab.js

@@ -18,6 +18,7 @@
 
 const ComputedArtifact = require('./computed-artifact');
 const log = require('lighthouse-logger');
+const Sentry = require('../../lib/sentry');
 
 // Bring in web-inspector for side effect of adding [].stableSort
 // See https://github.com/GoogleChrome/lighthouse/pull/2415
@@ -76,6 +77,9 @@ class TraceOfTab extends ComputedArtifact {
     // In this case, we'll use the last firstMeaningfulPaintCandidate we can find.
     // However, if no candidates were found (a bogus trace, likely), we fail.
     if (!firstMeaningfulPaint) {
+      // Track this with Sentry since it's likely a bug we should investigate.
+      Sentry.captureMessage('No firstMeaningfulPaint found, using fallback');
+
       const fmpCand = 'firstMeaningfulPaintCandidate';
       fmpFellBack = true;
       log.verbose('trace-of-tab', `No firstMeaningfulPaint found, falling back to last ${fmpCand}`);

lighthouse-core/gather/gatherers/dobetterweb/optimized-images.js

@@ -12,6 +12,7 @@
 
 const Gatherer = require('../gatherer');
 const URL = require('../../../lib/url-shim');
+const Sentry = require('../../../lib/sentry');
 
 const JPEG_QUALITY = 0.92;
 const WEBP_QUALITY = 0.85;
@@ -161,7 +162,16 @@ class OptimizedImages extends Gatherer {
     return imageRecords.reduce((promise, record) => {
       return promise.then(results => {
         return this.calculateImageStats(driver, record)
-          .catch(err => ({failed: true, err}))
+          .catch(err => {
+            // Track this with Sentry since these errors aren't surfaced anywhere else, but we don't
+            // want to tank the entire run due to a single image.
+            Sentry.captureException(err, {
+              tags: {gatherer: 'OptimizedImages'},
+              extra: {imageUrl: URL.elideDataURI(record.url)},
+              level: 'warning',
+            });
+            return {failed: true, err};
+          })
           .then(stats => {
             if (!stats) {
               return results;