diff --git a/lighthouse-core/lib/lh-error.js b/lighthouse-core/lib/lh-error.js index 9409f0db73e6..ae649ad86205 100644 --- a/lighthouse-core/lib/lh-error.js +++ b/lighthouse-core/lib/lh-error.js @@ -12,18 +12,20 @@ const strings = require('./strings'); * @property {string} code * @property {string} message * @property {RegExp} [pattern] + * @property {boolean} [lhrRuntimeError] True if it should appear in the top-level LHR.runtimeError property. */ class LighthouseError extends Error { /** * @param {LighthouseErrorDefinition} errorDefinition - * @param {Record=} properties + * @param {Record=} properties */ constructor(errorDefinition, properties) { super(errorDefinition.code); this.name = 'LHError'; this.code = errorDefinition.code; this.friendlyMessage = errorDefinition.message; + this.lhrRuntimeError = !!errorDefinition.lhrRuntimeError; if (properties) Object.assign(this, properties); Error.captureStackTrace(this, LighthouseError); @@ -75,17 +77,52 @@ class LighthouseError extends Error { const ERRORS = { // Screenshot/speedline errors - NO_SPEEDLINE_FRAMES: {code: 'NO_SPEEDLINE_FRAMES', message: strings.didntCollectScreenshots}, - SPEEDINDEX_OF_ZERO: {code: 'SPEEDINDEX_OF_ZERO', message: strings.didntCollectScreenshots}, - NO_SCREENSHOTS: {code: 'NO_SCREENSHOTS', message: strings.didntCollectScreenshots}, - INVALID_SPEEDLINE: {code: 'INVALID_SPEEDLINE', message: strings.didntCollectScreenshots}, + NO_SPEEDLINE_FRAMES: { + code: 'NO_SPEEDLINE_FRAMES', + message: strings.didntCollectScreenshots, + lhrRuntimeError: true, + }, + SPEEDINDEX_OF_ZERO: { + code: 'SPEEDINDEX_OF_ZERO', + message: strings.didntCollectScreenshots, + lhrRuntimeError: true, + }, + NO_SCREENSHOTS: { + code: 'NO_SCREENSHOTS', + message: strings.didntCollectScreenshots, + lhrRuntimeError: true, + }, + INVALID_SPEEDLINE: { + code: 'INVALID_SPEEDLINE', + message: strings.didntCollectScreenshots, + lhrRuntimeError: true, + }, // Trace parsing errors - NO_TRACING_STARTED: {code: 'NO_TRACING_STARTED', message: strings.badTraceRecording}, - NO_NAVSTART: {code: 'NO_NAVSTART', message: strings.badTraceRecording}, - NO_FCP: {code: 'NO_FCP', message: strings.badTraceRecording}, - NO_FMP: {code: 'NO_FMP', message: strings.badTraceRecording}, - NO_DCL: {code: 'NO_DCL', message: strings.badTraceRecording}, + NO_TRACING_STARTED: { + code: 'NO_TRACING_STARTED', + message: strings.badTraceRecording, + lhrRuntimeError: true, + }, + NO_NAVSTART: { + code: 'NO_NAVSTART', + message: strings.badTraceRecording, + lhrRuntimeError: true, + }, + NO_FCP: { + code: 'NO_FCP', + message: strings.badTraceRecording, + lhrRuntimeError: true, + }, + NO_DCL: { + code: 'NO_DCL', + message: strings.badTraceRecording, + lhrRuntimeError: true, + }, + NO_FMP: { + code: 'NO_FMP', + message: strings.badTraceRecording, + }, // TTI calculation failures FMP_TOO_LATE_FOR_FCPUI: {code: 'FMP_TOO_LATE_FOR_FCPUI', message: strings.pageLoadTookTooLong}, @@ -97,21 +134,36 @@ const ERRORS = { }, // Page load failures - NO_DOCUMENT_REQUEST: {code: 'NO_DOCUMENT_REQUEST', message: strings.pageLoadFailed}, - FAILED_DOCUMENT_REQUEST: {code: 'FAILED_DOCUMENT_REQUEST', message: strings.pageLoadFailed}, + NO_DOCUMENT_REQUEST: { + code: 'NO_DOCUMENT_REQUEST', + message: strings.pageLoadFailed, + lhrRuntimeError: true, + }, + FAILED_DOCUMENT_REQUEST: { + code: 'FAILED_DOCUMENT_REQUEST', + message: strings.pageLoadFailed, + lhrRuntimeError: true, + }, // Protocol internal failures TRACING_ALREADY_STARTED: { code: 'TRACING_ALREADY_STARTED', message: strings.internalChromeError, pattern: /Tracing.*started/, + lhrRuntimeError: true, }, PARSING_PROBLEM: { code: 'PARSING_PROBLEM', message: strings.internalChromeError, pattern: /Parsing problem/, + lhrRuntimeError: true, + }, + READ_FAILED: { + code: 'READ_FAILED', + message: strings.internalChromeError, + pattern: /Read failed/, + lhrRuntimeError: true, }, - READ_FAILED: {code: 'READ_FAILED', message: strings.internalChromeError, pattern: /Read failed/}, // Protocol timeout failures REQUEST_CONTENT_TIMEOUT: { @@ -122,5 +174,7 @@ const ERRORS = { /** @type {Record} */ LighthouseError.errors = ERRORS; +LighthouseError.NO_ERROR = 'NO_ERROR'; +LighthouseError.UNKNOWN_ERROR = 'UNKNOWN_ERROR'; module.exports = LighthouseError; diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js index d1d7aa2bd54c..77605732f1c3 100644 --- a/lighthouse-core/runner.js +++ b/lighthouse-core/runner.js @@ -18,6 +18,7 @@ const path = require('path'); const URL = require('./lib/url-shim'); const Sentry = require('./lib/sentry'); const generateReport = require('./report/report-generator').generateReport; +const LHError = require('./lib/lh-error.js'); /** @typedef {import('./gather/connections/connection.js')} Connection */ /** @typedef {import('./config/config.js')} Config */ @@ -131,6 +132,7 @@ class Runner { requestedUrl: requestedUrl, finalUrl: artifacts.URL.finalUrl, runWarnings: lighthouseRunWarnings, + runtimeError: Runner.getArtifactRuntimeError(artifacts), audits: resultsById, configSettings: settings, categories, @@ -291,6 +293,31 @@ class Runner { return auditResult; } + /** + * Returns first runtimeError found in artifacts. + * @param {LH.Artifacts} artifacts + * @return {LH.Result['runtimeError']} + */ + static getArtifactRuntimeError(artifacts) { + for (const possibleErrorArtifact of Object.values(artifacts)) { + if (possibleErrorArtifact instanceof LHError && possibleErrorArtifact.lhrRuntimeError) { + const errorMessage = possibleErrorArtifact.friendlyMessage ? + `${possibleErrorArtifact.friendlyMessage} (${possibleErrorArtifact.message})` : + possibleErrorArtifact.message; + + return { + code: possibleErrorArtifact.code, + message: errorMessage, + }; + } + } + + return { + code: LHError.NO_ERROR, + message: '', + }; + } + /** * Returns list of audit names for external querying. * @return {Array} diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json index 0a93fa9f746b..84e31d7ecfe8 100644 --- a/lighthouse-core/test/results/sample_v2.json +++ b/lighthouse-core/test/results/sample_v2.json @@ -10,6 +10,10 @@ "requestedUrl": "http://localhost:10200/dobetterweb/dbw_tester.html", "finalUrl": "http://localhost:10200/dobetterweb/dbw_tester.html", "runWarnings": [], + "runtimeError": { + "code": "NO_ERROR", + "message": "" + }, "audits": { "is-on-https": { "id": "is-on-https", diff --git a/lighthouse-core/test/runner-test.js b/lighthouse-core/test/runner-test.js index 274a2067ca71..3bc17cea57b4 100644 --- a/lighthouse-core/test/runner-test.js +++ b/lighthouse-core/test/runner-test.js @@ -10,12 +10,14 @@ const GatherRunner = require('../gather/gather-runner'); const driverMock = require('./gather/fake-driver'); const Config = require('../config/config'); const Audit = require('../audits/audit'); +const Gatherer = require('../gather/gatherers/gatherer.js'); const assetSaver = require('../lib/asset-saver'); const fs = require('fs'); const assert = require('assert'); const path = require('path'); const sinon = require('sinon'); const rimraf = require('rimraf'); +const LHError = require('../lib/lh-error.js'); /* eslint-env jest */ @@ -605,6 +607,41 @@ describe('Runner', () => { }); }); + it('includes a top-level runtimeError when a gatherer throws one', async () => { + const NO_FCP = LHError.errors.NO_FCP; + class RuntimeErrorGatherer extends Gatherer { + afterPass() { + throw new LHError(NO_FCP); + } + } + class WarningAudit extends Audit { + static get meta() { + return { + id: 'test-audit', + title: 'A test audit', + description: 'An audit for testing', + requiredArtifacts: ['RuntimeErrorGatherer'], + }; + } + static audit() { + throw new Error('Should not get here'); + } + } + + const config = new Config({ + passes: [{gatherers: [RuntimeErrorGatherer]}], + audits: [WarningAudit], + }); + const {lhr} = await Runner.run(null, {url: 'https://example.com/', config, driverMock}); + + // Audit error included the runtimeError + assert.strictEqual(lhr.audits['test-audit'].scoreDisplayMode, 'error'); + assert.ok(lhr.audits['test-audit'].errorMessage.includes(NO_FCP.code)); + // And it bubbled up to the runtimeError. + assert.strictEqual(lhr.runtimeError.code, NO_FCP.code); + assert.ok(lhr.runtimeError.message.includes(NO_FCP.code)); + }); + it('can handle array of outputs', async () => { const url = 'https://example.com'; const config = new Config({ diff --git a/lighthouse-extension/app/src/lighthouse-ext-background.js b/lighthouse-extension/app/src/lighthouse-ext-background.js index 253f51f31a49..fcb92a09a83e 100644 --- a/lighthouse-extension/app/src/lighthouse-ext-background.js +++ b/lighthouse-extension/app/src/lighthouse-ext-background.js @@ -11,6 +11,7 @@ const background = require('./lighthouse-background'); const ExtensionProtocol = require('../../../lighthouse-core/gather/connections/extension'); const log = require('lighthouse-logger'); const assetSaver = require('../../../lighthouse-core/lib/asset-saver.js'); +const LHError = require('../../../lighthouse-core/lib/lh-error.js'); /** @type {Record<'mobile'|'desktop', LH.Config.Json>} */ const LR_PRESETS = { @@ -111,14 +112,33 @@ async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs, config.settings.onlyCategories = categoryIDs; } - const results = await lighthouse(url, flags, config, connection); - if (!results) return; + try { + const results = await lighthouse(url, flags, config, connection); + if (!results) return; - if (logAssets) { - await assetSaver.logAssets(results.artifacts, results.lhr.audits); - } + if (logAssets) { + await assetSaver.logAssets(results.artifacts, results.lhr.audits); + } + return results.report; + } catch (err) { + // If an error ruined the entire lighthouse run, attempt to return a meaningful error. + let runtimeError; + if (!(err instanceof LHError) || !err.lhrRuntimeError) { + runtimeError = { + code: LHError.UNKNOWN_ERROR, + message: `Unknown error encountered with message '${err.message}'`, + }; + } else { + runtimeError = { + code: err.code, + message: err.friendlyMessage ? + `${err.friendlyMessage} (${err.message})` : + err.message, + }; + } - return results.report; + return JSON.stringify({runtimeError}, null, 2); + } } /** @@ -222,28 +242,32 @@ if ('chrome' in window && chrome.runtime) { } if (typeof module !== 'undefined' && module.exports) { - // Export for popup.js to import types. We don't want tsc to infer an index - // type, so use exports instead of module.exports. - exports.runLighthouseInExtension = runLighthouseInExtension; - exports.getDefaultCategories = background.getDefaultCategories; - exports.isRunning = isRunning; - exports.listenForStatus = listenForStatus; - exports.saveSettings = saveSettings; - exports.loadSettings = loadSettings; + // Export for importing types into popup.js, require()ing into unit tests. + module.exports = { + runLighthouseInExtension, + runLighthouseInLR, + getDefaultCategories: background.getDefaultCategories, + isRunning, + listenForStatus, + saveSettings, + loadSettings, + }; } -// Expose on window for extension, other consumers of file. -// @ts-ignore -window.runLighthouseInExtension = runLighthouseInExtension; -// @ts-ignore -window.runLighthouseInLR = runLighthouseInLR; -// @ts-ignore -window.getDefaultCategories = background.getDefaultCategories; -// @ts-ignore -window.isRunning = isRunning; -// @ts-ignore -window.listenForStatus = listenForStatus; -// @ts-ignore -window.loadSettings = loadSettings; -// @ts-ignore -window.saveSettings = saveSettings; +// Expose on window for extension, other browser-residing consumers of file. +if (typeof window !== 'undefined') { + // @ts-ignore + window.runLighthouseInExtension = runLighthouseInExtension; + // @ts-ignore + window.runLighthouseInLR = runLighthouseInLR; + // @ts-ignore + window.getDefaultCategories = background.getDefaultCategories; + // @ts-ignore + window.isRunning = isRunning; + // @ts-ignore + window.listenForStatus = listenForStatus; + // @ts-ignore + window.loadSettings = loadSettings; + // @ts-ignore + window.saveSettings = saveSettings; +} diff --git a/lighthouse-extension/test/app/src/lighthouse-ext-background-test.js b/lighthouse-extension/test/app/src/lighthouse-ext-background-test.js new file mode 100644 index 000000000000..c66858d8e6a9 --- /dev/null +++ b/lighthouse-extension/test/app/src/lighthouse-ext-background-test.js @@ -0,0 +1,57 @@ +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const assert = require('assert'); +const lhBackground = require('../../../app/src/lighthouse-ext-background.js'); +const LHError = require('../../../../lighthouse-core/lib/lh-error.js'); + +/* eslint-env mocha */ + +describe('lighthouse-ext-background', () => { + describe('#runLighthouseInLR', () => { + it('returns a runtimeError LHR when lighthouse throws a runtimeError', async () => { + const connectionError = new LHError(LHError.errors.FAILED_DOCUMENT_REQUEST); + assert.strictEqual(connectionError.lhrRuntimeError, true); + const mockConnection = { + async connect() { + throw connectionError; + }, + async disconnect() {}, + async sendCommand() {}, + on() {}, + }; + const url = 'https://example.com'; + const output = 'json'; + + const result = await lhBackground.runLighthouseInLR(mockConnection, url, {output}, {}); + const parsedResult = JSON.parse(result); + assert.strictEqual(parsedResult.runtimeError.code, connectionError.code); + assert.ok(parsedResult.runtimeError.message.includes(connectionError.friendlyMessage)); + }); + + it('returns an unknown-runtimeError LHR when lighthouse throws an unknown error', async () => { + const errorMsg = 'Errors are the best!'; + const connectionError = new Error(errorMsg); + assert.strictEqual(connectionError.lhrRuntimeError, undefined); + const mockConnection = { + async connect() { + throw connectionError; + }, + async disconnect() {}, + async sendCommand() {}, + on() {}, + }; + const url = 'https://example.com'; + const output = 'json'; + + const result = await lhBackground.runLighthouseInLR(mockConnection, url, {output}, {}); + const parsedResult = JSON.parse(result); + assert.strictEqual(parsedResult.runtimeError.code, LHError.UNKNOWN_ERROR); + assert.ok(parsedResult.runtimeError.message.includes(errorMsg)); + }); + }); +}); diff --git a/typings/lhr.d.ts b/typings/lhr.d.ts index 09cf635bf2fb..4245aff3272a 100644 --- a/typings/lhr.d.ts +++ b/typings/lhr.d.ts @@ -4,6 +4,8 @@ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +import LHError = require('../lighthouse-core/lib/lh-error.js'); + declare global { module LH { export type I18NMessageEntry = string | {path: string, values: any}; @@ -50,6 +52,8 @@ declare global { configSettings: Config.Settings; /** List of top-level warnings for this Lighthouse run. */ runWarnings: string[]; + /** A top-level error message that, if present, indicates a serious enough problem that this Lighthouse result may need to be discarded. */ + runtimeError: {code: string, message: string}; /** The User-Agent string of the browser used run Lighthouse for these results. */ userAgent: string; /** Information about the environment in which Lighthouse was run. */