core(errors-in-console): include runtime exceptions (#3494)

lighthouse-cli/test/fixtures/dobetterweb/dbw_tester.html

@@ -35,6 +35,9 @@
 <!-- FAIL: block rendering -->
 <script src="./dbw_tester.js"></script>
 
+<!-- FAIL(errors-in-console): exception thrown -->
+<script type="text/javascript">throw new Error('An error');</script>
+
 <style>
   body {
     color: #000;

lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js

@@ -15,11 +15,11 @@ module.exports = [
     audits: {
       'errors-in-console': {
         score: false,
-        rawValue: 3,
-        displayValue: '3',
+        rawValue: 4,
+        displayValue: '4',
         details: {
           items: {
-            length: 3,
+            length: 4,
           },
         },
       },

lighthouse-core/audits/errors-in-console.js

@@ -24,7 +24,7 @@ class ErrorLogs extends Audit {
       helpText: 'Errors logged to the console indicate unresolved problems. ' +
         'They can come from network request failures and other browser concerns.',
       failureDescription: 'Browser errors were logged to the console',
-      requiredArtifacts: ['ChromeConsoleMessages'],
+      requiredArtifacts: ['ChromeConsoleMessages', 'RuntimeExceptions'],
     };
   }
 
@@ -33,14 +33,29 @@ class ErrorLogs extends Audit {
    * @return {!AuditResult}
    */
   static audit(artifacts) {
-    const entries = artifacts.ChromeConsoleMessages;
-    const tableRows = entries.filter(log => log.entry.level === 'error').map(item => {
-      return {
-        source: item.entry.source,
-        description: item.entry.text,
-        url: item.entry.url,
-      };
-    });
+    const consoleEntries = artifacts.ChromeConsoleMessages;
+    const runtimeExceptions = artifacts.RuntimeExceptions;
+    const consoleRows =
+      consoleEntries.filter(log => log.entry && log.entry.level === 'error')
+      .map(item => {
+        return {
+          source: item.entry.source,
+          description: item.entry.text,
+          url: item.entry.url,
+        };
+      });
+
+    const runtimeExRows =
+      runtimeExceptions.filter(entry => entry.exceptionDetails !== undefined)
+      .map(entry => {
+        return {
+          source: 'Runtime.exception',
+          description: entry.exceptionDetails.exception.description,
+          url: entry.exceptionDetails.url,
+        };
+      });
+
+    const tableRows = consoleRows.concat(runtimeExRows);
 
     const headings = [
       {key: 'url', itemType: 'url', text: 'URL'},

lighthouse-core/config/default.js

@@ -22,6 +22,7 @@ module.exports = {
       'viewport-dimensions',
       'theme-color',
       'manifest',
+      'runtime-exceptions',
       'chrome-console-messages',
       'image-usage',
       'accessibility',

lighthouse-core/gather/gatherers/runtime-exceptions.js

@@ -0,0 +1,38 @@
+/**
+ * @license Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+/**
+ * @fileoverview Gathers runtime exceptions.
+ */
+
+'use strict';
+
+const Gatherer = require('./gatherer');
+
+class RuntimeExceptions extends Gatherer {
+  constructor() {
+    super();
+    this._exceptions = [];
+    this._onRuntimeExceptionThrown = this.onRuntimeExceptionThrown.bind(this);
+  }
+
+  onRuntimeExceptionThrown(entry) {
+    this._exceptions.push(entry);
+  }
+
+  beforePass(options) {
+    const driver = options.driver;
+    driver.on('Runtime.exceptionThrown', this._onRuntimeExceptionThrown);
+  }
+
+  afterPass(options) {
+    return Promise.resolve()
+      .then(_ => options.driver.off('Runtime.exceptionThrown', this._onRuntimeExceptionThrown))
+      .then(_ => this._exceptions);
+  }
+}
+
+module.exports = RuntimeExceptions;

lighthouse-core/test/audits/errors-in-console-test.js

@@ -14,6 +14,7 @@ describe('Console error logs audit', () => {
   it('passes when no console messages were found', () => {
     const auditResult = ErrorLogsAudit.audit({
       ChromeConsoleMessages: [],
+      RuntimeExceptions: [],
     });
     assert.equal(auditResult.rawValue, 0);
     assert.equal(auditResult.score, true);
@@ -32,6 +33,7 @@ describe('Console error logs audit', () => {
           },
         },
       ],
+      RuntimeExceptions: [],
     });
     assert.equal(auditResult.rawValue, 0);
     assert.equal(auditResult.score, true);
@@ -57,10 +59,31 @@ describe('Console error logs audit', () => {
           },
         },
       ],
+      RuntimeExceptions: [{
+        'timestamp': 1506535813608.003,
+        'exceptionDetails': {
+          'url': 'http://example.com/fancybox.js',
+          'stackTrace': {
+            'callFrames': [
+              {
+                'url': 'http://example.com/fancybox.js',
+                'lineNumber': 28,
+                'columnNumber': 20,
+              },
+            ],
+          },
+          'exception': {
+            'className': 'TypeError',
+            'description': 'TypeError: Cannot read property \'msie\' of undefined',
+          },
+          'executionContextId': 3,
+        },
+      }],
     });
-    assert.equal(auditResult.rawValue, 2);
+
+    assert.equal(auditResult.rawValue, 3);
     assert.equal(auditResult.score, false);
-    assert.equal(auditResult.details.items.length, 2);
+    assert.equal(auditResult.details.items.length, 3);
     assert.equal(auditResult.details.items[0][0].type, 'url');
     assert.equal(auditResult.details.items[0][0].text, 'http://www.example.com/favicon.ico');
     assert.equal(auditResult.details.items[0][1].type, 'text');
@@ -71,6 +94,12 @@ describe('Console error logs audit', () => {
     assert.equal(auditResult.details.items[1][1].type, 'text');
     assert.equal(auditResult.details.items[1][1].text,
       'WebSocket connection failed: Unexpected response code: 500');
+    assert.equal(auditResult.details.items[2][0].type, 'url');
+    assert.equal(auditResult.details.items[2][0].text,
+      'http://example.com/fancybox.js');
+    assert.equal(auditResult.details.items[2][1].type, 'text');
+    assert.equal(auditResult.details.items[2][1].text,
+      'TypeError: Cannot read property \'msie\' of undefined');
   });
 
   it('handle the case when some logs fields are undefined', () => {
@@ -82,6 +111,7 @@ describe('Console error logs audit', () => {
           },
         },
       ],
+      RuntimeExceptions: [],
     });
     assert.equal(auditResult.rawValue, 1);
     assert.equal(auditResult.score, false);

lighthouse-core/test/gather/gatherers/runtime-exceptions-test.js

@@ -0,0 +1,72 @@
+/**
+* @license Copyright 2017 Google Inc. All Rights Reserved.
+* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env mocha */
+
+const RuntimeExceptionsGatherer = require('../../../gather/gatherers/runtime-exceptions');
+const assert = require('assert');
+
+const mockDriver = {
+  off() {},
+};
+
+const wrapSendCommand = (mockDriver, runtimeEx) => {
+  mockDriver = Object.assign({}, mockDriver);
+
+  mockDriver.on = (name, cb) => {
+    if (name === 'Runtime.exceptionThrown') {
+      cb(runtimeEx);
+    }
+  };
+
+  mockDriver.sendCommand = () => {
+    return Promise.resolve();
+  };
+
+  return mockDriver;
+};
+
+describe('RuntimeExceptions', () => {
+  it('captures the exceptions raised', () => {
+    const runtimeExceptionsGatherer = new RuntimeExceptionsGatherer();
+    const runtimeEx =
+      {
+        'timestamp': 1506535813608.003,
+        'exceptionDetails': {
+          'url': 'http://www.example.com/fancybox.js',
+          'stackTrace': {
+            'callFrames': [
+              {
+                'url': 'http://www.example.com/fancybox.js',
+                'lineNumber': 28,
+                'columnNumber': 20,
+              },
+            ],
+          },
+          'exception': {
+            'className': 'TypeError',
+            'description': 'TypeError: Cannot read property \'msie\' of undefined',
+          },
+          'executionContextId': 3,
+        },
+      };
+
+    const options = {
+      driver: wrapSendCommand(mockDriver, runtimeEx),
+    };
+
+    return Promise.resolve(
+      runtimeExceptionsGatherer.beforePass(options))
+      .then(_ => runtimeExceptionsGatherer.afterPass(options))
+      .then((artifact) => {
+        assert.equal(artifact[0].exceptionDetails.exception.description,
+          `TypeError: Cannot read property 'msie' of undefined`);
+        assert.equal(artifact[0].exceptionDetails.url,
+          'http://www.example.com/fancybox.js');
+      });
+  });
+});