misc(scripts): add trace/devtoolslog minification scripts #5237

patrickhulce · 2018-05-17T00:16:02Z

phase 1 of lantern accuracy checks per commit

does a pretty awesome job of minifying traces from light/medium sites for inclusion, OK job of minifying the devtoolslog

example.com: 526KB -> 28KB, 7KB -> 4KB
paulirish.com: 2.3MB -> 151KB, 147KB -> 47KB

does okish on heavy sites like CNN

cnn.com: 30MB ->3.8MB, 2.9MB -> 1.3MB

paulirish · 2018-05-19T01:53:08Z

package.json

@@ -63,7 +63,8 @@
    "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --throttling-method=devtools --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing",
    "diff:sample-json": "bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh",
    "update:crdp-typings": "node lighthouse-core/scripts/extract-crdp-mapping.js",
-    "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js"
+    "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js",
+    "minify-latest-run": "./lighthouse-core/scripts/lantern/minify-trace.js ./latest-run/defaultPass.trace.json ./latest-run/defaultPass.trace.json && ./lighthouse-core/scripts/lantern/minify-devtoolslog.js ./latest-run/defaultPass.devtoolslog.json ./latest-run/defaultPass.devtoolslog.json"


latest-run/defaultPass.trace.json is in there twice. same with the dtlog.

if it's going to do it in place, maybe make the default output just be the input? Would make it a lot easier to read here, at least (otoh you might be sad that it automatically wrote over your saved trace, but I don't foresee many folks running this script independently)

I'd prefer not to make the default mode be inplace, I can put the latest-run files into different files? defaultPass.trace.min.json?

that's fine (either way, really). It was mostly a comment on npm scripts being generally unreadable and this with the doubled-up everything is especially hard to read :)

paulirish · 2018-05-19T01:54:38Z

lighthouse-core/scripts/lantern/minify-trace.js

+ * @param {LH.TraceEvent[]} events
+ */
+function filterTraceEvents(events) {
+  const startedInPageEvt = events.find(evt => evt.name === 'TracingStartedInPage');


lucky you: #5271 :)

speaking of which, is there some way to share all this logic with trace-of-tab so things like this (and what's considered "important") are always kept in sync?

yeah I'd want #5271 to move the logic into tracing-process for mainthread identification and then I can just reuse it :D

ok, so...future TODO/file an issue/wait until something breaks? :)

TODO it is!

paulirish · 2018-05-19T01:56:23Z

lighthouse-core/scripts/lantern/minify-trace.js

+  });
+
+  // Filter down the screenshots to key moments + 2fps animations
+  const screenshotTimestamps = filtered.filter(evt => evt.name === 'Screenshot').map(evt => evt.ts);


can you break each of these techniques into its own fn. i might want to use just the screenshots one in the future.

brendankenny · 2018-05-21T18:33:33Z

lighthouse-core/scripts/lantern/minify-devtoolslog.js

+  'x-robots-tag',
+]);
+
+/** @param {any} headers */


LH.Crdp.Network.Headers (which is just Record<string, string>)? Or I guess could be Partial<LH.Crdp.Network.Headers> since setting to undefined below?

brendankenny · 2018-05-21T18:33:56Z

lighthouse-core/scripts/lantern/minify-devtoolslog.js

+  obj.url = obj.url.replace(/^(data:.*?base64,).*/, '$1FILLER');
+}
+
+/** @param {LH.Crdp.Network.ResponseReceivedEvent['response']} [response] */


LH.Crdp.Network.Response?

brendankenny · 2018-05-21T18:43:31Z

lighthouse-core/scripts/lantern/minify-devtoolslog.js

+  /** @type {any} */
+  const timing = response.timing || {};
+  for (const [k, v] of Object.entries(timing)) {
+    if (v === -1) timing[k] = undefined;


how much does this end up mattering since the values will be just -1, not some unconstrained payload?

just eliminates most of the useless keys here, are you saying you'd rather keep them?

brendankenny · 2018-05-21T18:48:35Z

lighthouse-core/scripts/lantern/minify-trace.js

+ * @param {LH.TraceEvent[]} events
+ */
+function filterTraceEvents(events) {
+  const startedInPageEvt = events.find(evt => evt.name === 'TracingStartedInPage');


speaking of which, is there some way to share all this logic with trace-of-tab so things like this (and what's considered "important") are always kept in sync?

brendankenny · 2018-05-21T18:49:59Z

package.json

@@ -63,7 +63,8 @@
    "update:sample-json": "node ./lighthouse-cli -A=./lighthouse-core/test/results/artifacts --throttling-method=devtools --output=json --output-path=./lighthouse-core/test/results/sample_v2.json http://localhost/dobetterweb/dbw_tester.html && node lighthouse-core/scripts/cleanup-LHR-for-diff.js ./lighthouse-core/test/results/sample_v2.json --only-remove-timing",
    "diff:sample-json": "bash lighthouse-core/scripts/assert-golden-lhr-unchanged.sh",
    "update:crdp-typings": "node lighthouse-core/scripts/extract-crdp-mapping.js",
-    "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js"
+    "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js",
+    "minify-latest-run": "./lighthouse-core/scripts/lantern/minify-trace.js ./latest-run/defaultPass.trace.json ./latest-run/defaultPass.trace.json && ./lighthouse-core/scripts/lantern/minify-devtoolslog.js ./latest-run/defaultPass.devtoolslog.json ./latest-run/defaultPass.devtoolslog.json"


if it's going to do it in place, maybe make the default output just be the input? Would make it a lot easier to read here, at least (otoh you might be sad that it automatically wrote over your saved trace, but I don't foresee many folks running this script independently)

brendankenny · 2018-05-21T18:51:11Z

lighthouse-core/scripts/lantern/minify-devtoolslog.js

+
+/* eslint-disable no-console */
+
+const fs = require('fs');


fileoverview for both of these scripts would be 👍 👍

patrickhulce · 2018-05-22T17:39:55Z

PTAL :)

brendankenny

LGTM

misc(scripts): add trace/devtoolslog minification scripts

67ec22d

paulirish deployed to PR staging May 17, 2018 00:16 View deployment

always keep screenshots

4d57981

paulirish deployed to PR staging May 17, 2018 03:10 View deployment

keep more events

0dcbd1b

paulirish deployed to PR staging May 17, 2018 15:57 View deployment

lint

2747717

paulirish deployed to PR staging May 17, 2018 16:00 View deployment

throttle screenshot frames

55a832d

paulirish deployed to PR staging May 17, 2018 16:36 View deployment

filter out events not in tasks

05e92bf

paulirish deployed to PR staging May 17, 2018 17:17 View deployment

patrickhulce mentioned this pull request May 18, 2018

misc(scripts): add lantern evaluation scripts #5257

Merged

paulirish reviewed May 19, 2018

View reviewed changes

brendankenny reviewed May 21, 2018

View reviewed changes

feedback

6a1685e

paulirish deployed to PR staging May 21, 2018 20:14 View deployment

todo

c138022

paulirish deployed to PR staging May 21, 2018 22:28 View deployment

brendankenny approved these changes May 22, 2018

View reviewed changes

paulirish approved these changes May 22, 2018

View reviewed changes

patrickhulce merged commit 40cf8da into master May 22, 2018

patrickhulce deleted the mini_traces branch May 22, 2018 19:11

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

misc(scripts): add trace/devtoolslog minification scripts #5237

misc(scripts): add trace/devtoolslog minification scripts #5237

patrickhulce commented May 17, 2018 •

edited

Loading

paulirish May 19, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018 •

edited

Loading

brendankenny May 21, 2018

paulirish May 19, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

paulirish May 19, 2018

patrickhulce May 21, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

brendankenny May 21, 2018

brendankenny May 21, 2018

brendankenny May 21, 2018

patrickhulce May 21, 2018

patrickhulce commented May 22, 2018

brendankenny left a comment

misc(scripts): add trace/devtoolslog minification scripts #5237

misc(scripts): add trace/devtoolslog minification scripts #5237

Conversation

patrickhulce commented May 17, 2018 • edited Loading

phase 1 of lantern accuracy checks per commit

Choose a reason for hiding this comment

Choose a reason for hiding this comment

patrickhulce May 21, 2018 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

patrickhulce commented May 22, 2018

brendankenny left a comment

Choose a reason for hiding this comment

patrickhulce commented May 17, 2018 •

edited

Loading

patrickhulce May 21, 2018 •

edited

Loading