diff --git a/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js b/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js index 97020f3d07cb..68e0c192d15f 100755 --- a/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js +++ b/lighthouse-core/scripts/lantern/assert-master-lantern-values-unchanged.js @@ -10,50 +10,60 @@ const fs = require('fs'); const path = require('path'); -const execSync = require('child_process').execSync; const constants = require('./constants'); +const chalk = require('chalk').default; const INPUT_PATH = process.argv[2] || constants.SITE_INDEX_WITH_GOLDEN_WITH_COMPUTED_PATH; const HEAD_PATH = path.resolve(process.cwd(), INPUT_PATH); const MASTER_PATH = constants.MASTER_COMPUTED_PATH; -const TMP_DIR = path.join(__dirname, '../../../.tmp'); -const TMP_HEAD_PATH = path.join(TMP_DIR, 'HEAD-for-diff.json'); -const TMP_MASTER_PATH = path.join(TMP_DIR, 'master-for-diff.json'); - -if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR); - if (!fs.existsSync(HEAD_PATH) || !fs.existsSync(MASTER_PATH)) { throw new Error('Usage $0 '); } -let exitCode = 0; - -try { - const computedResults = require(HEAD_PATH); - const expectedResults = require(MASTER_PATH); - - const sites = []; - for (const entry of computedResults.sites) { - const lanternValues = entry.lantern; - Object.keys(lanternValues).forEach(key => lanternValues[key] = Math.round(lanternValues[key])); - sites.push({url: entry.url, ...lanternValues}); - } - - fs.writeFileSync(TMP_HEAD_PATH, JSON.stringify({sites}, null, 2)); - fs.writeFileSync(TMP_MASTER_PATH, JSON.stringify(expectedResults, null, 2)); - - try { - execSync(`git --no-pager diff --color=always --no-index ${TMP_MASTER_PATH} ${TMP_HEAD_PATH}`); - console.log('✅ PASS No changes between expected and computed!'); - } catch (err) { - console.log('❌ FAIL Changes between expected and computed!\n'); - console.log(err.stdout.toString()); - exitCode = 1; - } -} finally { - fs.unlinkSync(TMP_HEAD_PATH); - fs.unlinkSync(TMP_MASTER_PATH); +const computedResults = require(HEAD_PATH); +const expectedResults = require(MASTER_PATH); + +/** @type {Array<{url: string, maxDiff: number, diffsForSite: Array}>} */ +const diffs = []; +for (const entry of computedResults.sites) { + // @ts-ignore - over-aggressive implicit any on candidate + const expectedLantern = expectedResults.sites.find(candidate => entry.url === candidate.url); + const actualLantern = entry.lantern; + + let maxDiff = 0; + /** @type {DiffForSite[]} */ + const diffsForSite = []; + Object.keys(actualLantern).forEach(metricName => { + const actual = Math.round(actualLantern[metricName]); + const expected = Math.round(expectedLantern[metricName]); + const diff = actual - expected; + if (Math.abs(diff) > 0) { + maxDiff = Math.max(maxDiff, Math.abs(diff)); + diffsForSite.push({metricName, actual, expected, diff}); + } + }); + + if (maxDiff > 0) diffs.push({url: entry.url, maxDiff, diffsForSite}); +} + +if (diffs.length) { + console.log(`❌ FAIL ${diffs.length} change(s) between expected and computed!\n`); + + diffs.sort((a, b) => b.maxDiff - a.maxDiff).forEach(site => { + console.log(chalk.magenta(site.url)); + site.diffsForSite.sort((a, b) => Math.abs(b.diff) - Math.abs(a.diff)).forEach(entry => { + const metric = ` - ${entry.metricName.padEnd(25)}`; + const diff = entry.diff > 0 ? chalk.yellow(`+${entry.diff}`) : chalk.cyan(`${entry.diff}`); + const actual = `${entry.actual} ${chalk.gray('(HEAD)')}`; + const expected = `${entry.expected} ${chalk.gray('(master)')}`; + console.log(`${metric}${diff}\t${actual}\tvs.\t${expected}`); + }); + }); + + process.exit(1); +} else { + console.log('✅ PASS No changes between expected and computed!'); } -process.exit(exitCode); +/** @typedef {{metricName: string, actual: number, expected: number, diff: number}} DiffForSite */ diff --git a/lighthouse-core/scripts/lantern/debug-url.js b/lighthouse-core/scripts/lantern/debug-url.js new file mode 100755 index 000000000000..5230acc10ec6 --- /dev/null +++ b/lighthouse-core/scripts/lantern/debug-url.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +const path = require('path'); +const execFileSync = require('child_process').execFileSync; +const constants = require('./constants'); + +const INPUT_URL = process.argv[2]; +if (!INPUT_URL) throw new Error('Usage $0: '); + +const SITE_INDEX_PATH = path.resolve(process.cwd(), constants.SITE_INDEX_WITH_GOLDEN_PATH); +const SITE_INDEX_DIR = path.dirname(SITE_INDEX_PATH); +const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js'); + +const siteIndex = require(SITE_INDEX_PATH); +// @ts-ignore - over-aggressive implicit any on site +const site = siteIndex.sites.find(site => site.url === INPUT_URL); +if (!site) throw new Error(`Could not find with site URL ${INPUT_URL}`); + +const trace = path.join(SITE_INDEX_DIR, site.unthrottled.tracePath); +const log = path.join(SITE_INDEX_DIR, site.unthrottled.devtoolsLogPath); +process.env.LANTERN_DEBUG = 'true'; +execFileSync('node', ['--inspect-brk', RUN_ONCE_PATH, trace, log], {stdio: 'inherit'}); diff --git a/lighthouse-core/scripts/lantern/run-once.js b/lighthouse-core/scripts/lantern/run-once.js index b434a5212d5a..c89d065c9827 100755 --- a/lighthouse-core/scripts/lantern/run-once.js +++ b/lighthouse-core/scripts/lantern/run-once.js @@ -6,21 +6,33 @@ */ 'use strict'; +const fs = require('fs'); const path = require('path'); -const LH_ROOT_DIR = path.join(__dirname, '../../../'); +const PredictivePerf = require('../../../lighthouse-core/audits/predictive-perf'); +const Runner = require('../../../lighthouse-core/runner'); +const Simulator = require('../../../lighthouse-core/lib/dependency-graph/simulator/simulator'); +const traceSaver = require('../../../lighthouse-core/lib/lantern-trace-saver'); if (process.argv.length !== 4) throw new Error('Usage $0 '); async function run() { - const PredictivePerf = require(path.join(LH_ROOT_DIR, 'lighthouse-core/audits/predictive-perf')); - const Runner = require(path.join(LH_ROOT_DIR, 'lighthouse-core/runner')); - - const traces = {defaultPass: require(process.argv[2])}; + const tracePath = require.resolve(process.argv[2]); + const traces = {defaultPass: require(tracePath)}; const devtoolsLogs = {defaultPass: require(process.argv[3])}; const artifacts = {traces, devtoolsLogs, ...Runner.instantiateComputedArtifacts()}; + // @ts-ignore - We don't need the full artifacts const result = await PredictivePerf.audit(artifacts); process.stdout.write(JSON.stringify(result.details.items[0], null, 2)); + + // Dump the TTI graph with simulated timings to a trace if LANTERN_DEBUG is enabled + const pessimisticTTINodeTimings = Simulator.ALL_NODE_TIMINGS.get('pessimisticInteractive'); + if (process.env.LANTERN_DEBUG && pessimisticTTINodeTimings) { + const outputTraceFile = path.basename(tracePath).replace(/.trace.json$/, '.lantern.trace.json'); + const outputTracePath = path.join(__dirname, '../../../.tmp', outputTraceFile); + const trace = traceSaver.convertNodeTimingsToTrace(pessimisticTTINodeTimings); + fs.writeFileSync(outputTracePath, JSON.stringify(trace, null, 2)); + } } run().catch(err => { diff --git a/package.json b/package.json index bc27eee5150e..a1c278b93047 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "@types/yargs": "^8.0.2", "babel-core": "^6.26.0", "bundlesize": "^0.14.4", + "chalk": "^2.4.1", "codecov": "^2.2.0", "commitizen": "^2.9.6", "conventional-changelog-cli": "^1.3.4", diff --git a/yarn.lock b/yarn.lock index d1091557e9e1..42e4d8c09465 100644 --- a/yarn.lock +++ b/yarn.lock @@ -303,6 +303,12 @@ ansi-styles@^3.1.0: dependencies: color-convert "^1.9.0" +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + dependencies: + color-convert "^1.9.0" + append-transform@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-0.4.0.tgz#d76ebf8ca94d276e247a36bad44a4b74ab611991" @@ -843,6 +849,14 @@ chalk@2.1.0, chalk@^2.0.0, chalk@^2.1.0: escape-string-regexp "^1.0.5" supports-color "^4.0.0" +chalk@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e" + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chrome-devtools-frontend@1.0.401423: version "1.0.401423" resolved "https://registry.yarnpkg.com/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.401423.tgz#32a89b8d04e378a494be3c8d63271703be1c04ea" @@ -2377,6 +2391,10 @@ has-flag@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + has-gulplog@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/has-gulplog/-/has-gulplog-0.1.0.tgz#6414c82913697da51590397dafb12f22967811ce" @@ -4795,6 +4813,12 @@ supports-color@^4.0.0: dependencies: has-flag "^2.0.0" +supports-color@^5.3.0: + version "5.4.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" + dependencies: + has-flag "^3.0.0" + symbol-tree@^3.2.1: version "3.2.2" resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.2.tgz#ae27db38f660a7ae2e1c3b7d1bc290819b8519e6"