core(gather): add new MainDocumentContent public artifact (#9781)

GoogleChrome · Oct 16, 2019 · d9e6499 · d9e6499
1 parent ba2f21b
commit d9e6499
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 14 deletions.
diff --git a/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/index-test.js.snap
@@ -1147,6 +1147,9 @@ Object {
         Object {
           "path": "iframe-elements",
         },
+        Object {
+          "path": "main-document-content",
+        },
         Object {
           "path": "dobetterweb/appcache",
         },

diff --git a/lighthouse-cli/test/smokehouse/dbw-config.js b/lighthouse-cli/test/smokehouse/dbw-config.js
@@ -10,18 +10,6 @@
  */
 module.exports = {
   extends: 'lighthouse:default',
-  settings: {
-    onlyCategories: [
-      'best-practices',
-    ],
-    onlyAudits: [
-      'dom-size',
-      'render-blocking-resources',
-      'errors-in-console',
-      'efficient-animated-content',
-      'apple-touch-icon', // pull in apple touch icon to test `LinkElements`
-    ],
-  },
   audits: [
     // Test the `ignoredPatterns` audit option.
     {path: 'errors-in-console', options: {ignoredPatterns: ['An ignored error']}},

diff --git a/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js b/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js
@@ -20,6 +20,7 @@ const expectations = [
       }, {
         id: 'wordpress',
       }],
+      MainDocumentContent: /^<!doctype html>.*DoBetterWeb Mega Tester.*aggressive-promise-polyfill.*<\/html>\n$/s,
       LinkElements: [
         {
           rel: 'stylesheet',

diff --git a/lighthouse-core/config/default-config.js b/lighthouse-core/config/default-config.js
@@ -136,6 +136,7 @@ const defaultConfig = {
       'meta-elements',
       'script-elements',
       'iframe-elements',
+      'main-document-content',
       'dobetterweb/appcache',
       'dobetterweb/doctype',
       'dobetterweb/domstats',

diff --git a/lighthouse-core/gather/gatherers/main-document-content.js b/lighthouse-core/gather/gatherers/main-document-content.js
@@ -0,0 +1,28 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Gatherer = require('./gatherer.js');
+const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
+
+/**
+ * Collects the content of the main html document.
+ */
+class MainDocumentContent extends Gatherer {
+  /**
+   * @param {LH.Gatherer.PassContext} passContext
+   * @param {LH.Gatherer.LoadData} loadData
+   * @return {Promise<LH.Artifacts['MainDocumentContent']>}
+   */
+  async afterPass(passContext, loadData) {
+    const mainResource = NetworkAnalyzer.findMainDocument(loadData.networkRecords, passContext.url);
+
+    const driver = passContext.driver;
+    return driver.getRequestContent(mainResource.requestId);
+  }
+}
+
+module.exports = MainDocumentContent;
diff --git a/lighthouse-core/test/results/artifacts/artifacts.json b/lighthouse-core/test/results/artifacts/artifacts.json
@@ -2113,5 +2113,6 @@
     "name": "html",
     "systemId": "",
     "publicId": ""
-  }
-}
+  },
+  "MainDocumentContent": "<!doctype html>\n<!--\n * Copyright 2016 Google Inc. All Rights Reserved.\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0\n * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n-->\n\n<!-- FAIL - appcache manifest. Note: Appcache cannot be toggled at runtime. -->\n<html manifest=\"clock.appcache\">\n<head>\n<title>DoBetterWeb Mega Tester... Of Death</title>\n<meta charset=\"utf-8\">\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1, minimum-scale=1\">\n\n<template id=\"links-blocking-first-paint-tmpl\">\n  <link rel=\"stylesheet\" href=\"./dbw_tester.css?scriptActivated&delay=200\"> <!-- PASS: initiator is script -->\n</template>\n\n<!-- AMP-style stylesheet script -->\n<!-- based on https://github.com/ampproject/amphtml/blob/38f1bdf4cc385f9a25cf2979abf215952880b876/src/font-stylesheet-timeout.js#L87-L103 -->\n<script type=\"text/javascript\" id=\"amp-style-styling-script\">\n  setTimeout(() => {\n    const stylesheet = document.getElementById('amp-style-link');\n    stylesheet.media = 'not-matching';\n    stylesheet.onload = () => stylesheet.media = 'screen';\n    stylesheet.parentNode.insertBefore(stylesheet, stylesheet.nextSibling);\n  }, 1500);\n</script>\n\n<!-- Note: these will only fail when using the static-server.js, which supports the ?delay=true param.\n     If you're using your own server, the resource will load instantly and the\n     stylesheets will be ignored for being below the threshold. -->\n<link rel=\"stylesheet\" href=\"./dbw_tester.css?delay=100\">\n<link rel=\"stylesheet\" href=\"./unknown404.css?delay=200\"> <!-- FAIL -->\n<link rel=\"stylesheet\" href=\"./dbw_tester.css?delay=2200\"> <!-- FAIL -->\n<link rel=\"stylesheet\" href=\"./dbw_disabled.css?delay=200&isdisabled\" disabled> <!-- PASS -->\n<link rel=\"import\" href=\"./dbw_partial_a.html?delay=200\"> <!-- FAIL -->\n<link rel=\"import\" href=\"./dbw_partial_b.html?delay=200&isasync\" async> <!-- PASS -->\n\n<!-- FAIL: render blocking but capped at 500ms -->\n<link id=\"amp-style-link\" rel=\"stylesheet\" href=\"./dbw_tester.css?delay=3000&capped\">\n<!-- PASS: preload that's activated later does not block rendering. -->\n<link rel=\"preload\" href=\"./dbw_tester.css?delay=2000&async=true\" as=\"style\" onload=\"this.rel = 'stylesheet'\">\n<!-- PASS: async stylesheet -->\n<link rel=\"stylesheet\" href=\"./dbw_tester.css?delay=3000&async=true\" disabled onload=\"this.disabled = false\">\n\n<!-- Alternate stylesheets should not show in TagsBlockingFirstPaint artifact -->\n<link rel=\"alternate stylesheet\" href=\"./empty.css\">\n\n<!-- FAIL: block rendering -->\n<script src=\"./dbw_tester.js\" id=\"dbw-tester-script\"></script>\n\n<!-- PASS: 'modules' are deferred by default and don't block rendering -->\n<script type=\"module\" src=\"./empty_module.js?delay=500\"></script>\n\n<!-- FAIL(errors-in-console): exception thrown -->\n<script type=\"text/javascript\" id=\"error-time\">throw new Error('A distinctive error');</script>\n\n<!-- Add non-functional script to appear to be a Wordpress page for Stack Packs. -->\n<script type=\"example\" src=\"fake-script-wp-includes\"></script>\n\n<style>\n  body {\n    color: #000;\n  }\n  .failedselector {\n    -webkit-box-flex: 1; /* FAIL */\n    box-flex: 1; /* FAIL */\n  }\n</style>\n\n<template id=\"old-flexbox-tmpl\">\n<style>\n  p, div {\n    background: red, green;\n    border-radius: 3px;\n    transform: translateZ(0);\n    display: box; /* FAIL */\n    transition: opacity 300ms ease-in-out;\n  }\n  .span {\n    transform: translate3d(0,0,0);\n    display: box; /* FAIL: old flexbox usage */\n  }\n  .span2,\n  .span3 {\n          display: box;  /* FAIL: old flexbox usage */\n    color:     red;\n  }\n  .span4,\n  .span5 { display:     box; /* FAIL - old flexbox, weird formatting */\n    box-shadow: 3px 3px 3px purple,\n                3px 3px 3px blue;\n  }\n  .span6 {\n    box-shadow: 3px 3px 3px purple,\n                3px 3px\n                3px green;\n  }\n</style>\n</template>\n\n<template id=\"deprecations-tmpl\">\n  <style>\n    body /deep/ div {\n      color: pink; /* FAIL - deprecation warning */\n    }\n  </style>\n</template>\n\n<!-- Force FCP to be ~5s out, necessary to make sure our render-blocking audits still work -->\n<!-- even when other forces were responsible for delaying the render. -->\n<script src=\"./fcp-delayer.js?delay=5000\"></script>\n</head>\n<body>\n\n<!-- Various sites like to assign HTMLElements a custom `matches` property. See issue #5934 -->\n\n<!-- EmbeddedContent will process these elements -->\n<object id=\"5934a\"></object>\n<object id=\"5934b\"></object>\n\n<div id=\"shadow-root-container\"></div>\n<script>\n  // DOM Size has had trouble with very specific shadow root traversal issues.\n  // Make sure it can handle reporting of the widest element being a shadow root.\n  const shadowRootContainer = document.getElementById('shadow-root-container');\n  const shadowRoot = shadowRootContainer.attachShadow({mode: 'open'});\n  for (let i = 0; i < 100; i++) {\n    const span = document.createElement('span');\n    shadowRoot.appendChild(span);\n  }\n</script>\n\n<script>\n  // Ensure gatherers still work when individual elements override '.matches'\n  document.getElementById(\"5934a\").matches = \"blahblah\";\n  Object.defineProperty(document.getElementById(\"5934b\"), 'matches', {\n    value: \"blahblah\"\n  });\n  // Ensure gatherers still work when the prototype is messed with\n  HTMLElement.prototype.matches = { value: \"blahblah\" };\n\n\n  // Ensure long-task collection still works when performance.now is redefined\n  window.performance.now = 'right now';\n</script>\n\n<div>\n  <h2>Do better web tester page</h2>\n  <span>Hi there!</span>\n\n  <!-- Invalid SVG with a link just to ensure the `LinkElements` gatherer can handle it. -->\n  <svg role=\"img\" class=\"social-facebook\" aria-labelledby=\"social-facebook-5\">\n    <title id=\"social-facebook-5\">Facebook</title>\n    <link rel=\"preload\" href=\"https://fonts.googleapis.com/made-up-url\"></link>\n    <use xlink:href=\"\"></use>\n  </svg>\n</div>\n\n<section id=\"touchmove-section\">touchmove section</section>\n\n<template id=\"noopener-links-tmpl\">\n  <!-- FAIL - does not use rel=\"noopener\" to open external link -->\n  <a href=\"https://www.google.com/\" target=\"_blank\">external link</a>\n  <!-- FAIL - does not use rel=\"noopener\" and has no href attribute, giving an\n    href value of '' when read, which will throw in a `new URL('')` constructor -->\n  <a target=\"_blank\">external link</a>\n  <!-- FAIL - external link that does have a rel attribute but it is not \"noopener\" -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"nofollow\">external link</a>\n  <!-- PASS - external link correctly uses rel=\"noopener\" and an additional rel value -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"noopener nofollow\">external link that uses rel noopener and another unrelated rel attribute</a>\n  <!-- PASS - external link correctly uses rel=\"noreferrer\" and an additional rel value -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"noreferrer nofollow\">external link that uses rel noreferrer and another unrelated rel attribute</a>\n  <!-- PASS - external link correctly uses rel=\"noopener\" -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"noopener\">external link that uses rel noopener</a>\n  <!-- PASS - external link correctly uses rel=\"noreferrer\" -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"noreferrer\">external link that uses rel noreferrer</a>\n  <!-- PASS - external link correctly uses rel=\"noopener\" and rel=\"noreferrer\" -->\n  <a href=\"https://www.google.com/\" target=\"_blank\" rel=\"noopener noreferrer\">external link that uses rel noopener and noreferrer</a>\n  <!-- PASS - internal link without rel=\"noopener\" -->\n  <a href=\"./doesnotexist\" target=\"_blank\">internal link is ok</a>\n  <!-- PASS - href uses javascript: -->\n  <a href=\"javascript:void(0)\" target=\"_blank\"></a>\n  <!-- PASS - href uses mailto: -->\n  <a href=\"mailto:inbox@email.com\" target=\"_blank\"></a>\n</template>\n\n<template id=\"password-inputs-can-be-pasted-into\">\n  <!-- FAIL - calls preventDefault on event -->\n  <input type=\"password\" onpaste=\"event.preventDefault();\" />\n  <!-- PASS -->\n  <input type=\"password\" />\n  <!-- FAIL - returns false in event handler -->\n  <input type=\"password\" onpaste=\"return false;\" />\n\n</template>\n\n<!-- FAIL(image-aspect-ratio): image is naturally 480x318 -->\n<img src=\"lighthouse-480x318.jpg\" width=\"480\" height=\"57\">\n<!-- PASS(image-aspect-ratio) -->\n<img src=\"lighthouse-480x318.jpg\" width=\"480\" height=\"318\">\n\n<!-- FAIL(efficient-animated-content): animated gif found -->\n<img src=\"lighthouse-rotating.gif\" width=\"811\" height=\"462\">\n\n<!-- FAIL(tap-targets): buttons too close together -->\n<style>\n.small-button {\n  display: block;\n  width: 200px;\n}\n</style>\n<button class=\"small-button\">Do something</button>\n<button class=\"small-button\">Do something else</button>\n\n<!-- Some websites overwrite the original Error object. The captureJSCallUsage function\n  relies on the native Error object and prepareStackTrace from V8. When overwriting the stack\n  property the e.stack inside the gatherer won't be in the correct format\n  https://github.com/GoogleChrome/lighthouse/issues/1194 -->\n<script>window.Error = function(error) { this.stack = 'stacktrace'; };</script>\n\n<!-- Does the automatic dialog suppression `driver.dismissJavaScriptDialogs()` work? -->\n<script>window.confirm('Is DBW Mega Tester the best site?')</script>\n\n<script>\nfunction stampTemplate(id, location) {\n  const template = document.querySelector('#' + id);\n  location.appendChild(template.content.cloneNode(true));\n}\n\nfunction dateNowTest() {\n  function helloDate() {\n    // FAIL - Date.now() usage in a function.\n    return Date.now();\n  }\n  helloDate();\n  const d = Date.now(); // FAIL\n  eval('Date.now()'); // FAIL\n  new Function('Date.now()')() // FAIL\n}\n\nfunction consoleTimeTest() {\n  // FAIL\n  console.time('arg1');\n  for (let i = 0; i < 2; i++) {\n    // FAIL\n    console.time('arg2');\n  }\n  console.timeEnd('arg1');\n  console.timeEnd('arg2');\n\n  eval(\"console.time('arg3')\"); // FAIL\n}\n\nfunction documentWriteTest() {\n  document.write('Hi'); // FAIL\n  document.write('There'); // FAIL\n  document.write('2.0!'); // FAIL\n}\n\nfunction passiveEventsListenerTest() {\n  // FAIL or PASS - field trial for document level defaulting to passive M71+\n  window.addEventListener('wheel', function(e) {\n    console.log('wheel');\n  });\n\n  // PASS - document level defaults to passive M56+\n  window.addEventListener('touchstart', function(e) {\n    console.log('touchstart');\n  });\n\n  // PASS - passive:false doesn't get a warning now. crbug.com/770208\n  window.addEventListener('mousewheel', function(e) {\n    console.log('mousewheel');\n  }, {passive: false});\n\n  // PASS - document level defaults to passive M56+\n  document.addEventListener('touchstart', function(e) {\n    console.log('touchstart document');\n  });\n\n  // PASS - document level defaults to passive M56+\n  document.body.addEventListener('touchmove', function(e) {\n    console.log('touchmove');\n  });\n\n  // FAIL\n  const el = document.querySelector('#touchmove-section');\n  el.addEventListener('touchmove', function(e) {\n    console.log('touchmove');\n  });\n\n  // PASS - calls preventDefault() but isnt passive;\n  document.addEventListener('touchstart', function(e) {\n\n    e.preventDefault(); // intentionally surrounded by whitespace.\n\n    console.log('touchstart - preventDefault called');\n  });\n\n  // PASS\n  document.body.addEventListener('touchstart', function(e) {\n    console.log('touchstart');\n  }, {passive: true});\n\n  // PASS\n  window.addEventListener('touchstart', function(e) {\n    console.log('touchstart passive');\n  }, {passive: true});\n\n  // PASS - another event on window that isn't wheel/touch\n  window.addEventListener('scroll', function(e) {\n    console.log('scroll');\n  });\n}\n\nfunction geolocationOnStartTest() {\n  navigator.geolocation.getCurrentPosition(function(position) {\n    // noop\n  });\n\n  const watchID = navigator.geolocation.watchPosition(function(position) {\n    navigator.geolocation.clearWatch(watchID);\n  });\n}\n\nfunction notificationOnStartTest() {\n  Notification.requestPermission().then(function () {\n    // noop\n  });\n}\n\nfunction oldCSSFlexboxTest() {\n  stampTemplate('old-flexbox-tmpl', document.body);\n}\n\nfunction linksBlockingFirstPaintTest() {\n  stampTemplate('links-blocking-first-paint-tmpl', document.head);\n}\n\nfunction noRelOpenLinksTest() {\n  stampTemplate('noopener-links-tmpl', document.body);\n}\n\nfunction passwordInputsCanBePastedIntoTest() {\n  stampTemplate('password-inputs-can-be-pasted-into', document.body);\n}\n\n\nfunction deprecationsTest() {\n  stampTemplate('deprecations-tmpl', document.head);\n\n  // TODO: some deprecated API messages are not currently being logged correctly.\n  // See: https://crbug.com/680832\n\n  // FAIL\n  const xhr = new XMLHttpRequest();\n  xhr.open('GET', 'dbw_tester.html', false);\n  xhr.send();\n\n  // One more failure in dbw_tester.js\n}\n\nfunction isOnHttps() {\n  // Check blob URL.\n  const blob = new Blob(['fake'])\n  const img = document.createElement('img');\n  img.src = URL.createObjectURL(blob);\n  document.body.appendChild(img); // PASS\n}\n\n// Figure out which tests to fun.\nconst params = new URLSearchParams(location.search);\nif (location.search === '') {\n  documentWriteTest();\n  dateNowTest();\n  consoleTimeTest();\n  passiveEventsListenerTest();\n  geolocationOnStartTest();\n  notificationOnStartTest();\n  linksBlockingFirstPaintTest();\n  noRelOpenLinksTest();\n  // oldCSSFlexboxTest();\n  deprecationsTest();\n  passwordInputsCanBePastedIntoTest();\n  isOnHttps();\n} else {\n  if (params.has('documentWrite')) {\n    documentWriteTest();\n  }\n  if (params.has('dateNow')) {\n    dateNowTest();\n  }\n  if (params.has('consoleTime')) {\n    consoleTimeTest();\n  }\n  if (params.has('passiveEvents')) {\n    passiveEventsListenerTest();\n  }\n  if (params.has('geolocation')) {\n    geolocationOnStartTest();\n  }\n  if (params.has('notifications')) {\n    notificationOnStartTest();\n  }\n  if (params.has('linksblockfp')) {\n    linksBlockingFirstPaintTest();\n  }\n  if (params.has('relnoopener')) {\n    noRelOpenLinksTest();\n  }\n  // if (params.has('oldcssflexbox')) {\n  //   oldCSSFlexboxTest();\n  // }\n  if (params.has('deprecations')) {\n    deprecationsTest();\n  }\n  if (params.has('passwordinputs')) {\n    passwordInputsCanBePastedIntoTest();\n  }\n  if (params.has('isonhttps')) {\n    isOnHttps();\n  }\n}\n</script>\n\n<!-- Including this script validates that driver.evaluateAsync() still works even with\n  window.Promise being mucked with. See file source for details.\n  Also, script is located down here to avoid being flagged in render-blocking-resources. -->\n<script src=\"./third_party/aggressive-promise-polyfill.js\"></script>\n\n<!-- FAIL(is-on-https): requires a non-localhost http file\n     FAIL(no-vulnerable-libraries): Loads a vulnerable JS library -->\n<script src=\"http://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js\"></script>\n\n</body>\n</html>\n"
+}
diff --git a/types/artifacts.d.ts b/types/artifacts.d.ts
@@ -63,6 +63,8 @@ declare global {
       ConsoleMessages: Crdp.Log.EntryAddedEvent[];
       /** All the iframe elements in the page.*/
       IFrameElements: Artifacts.IFrameElement[];
+      /** The contents of the main HTML document network resource. */
+      MainDocumentContent: string;
       /** Information on size and loading for all the images in the page. Natural size information for `picture` and CSS images is only available if the image was one of the largest 50 images. */
       ImageElements: Artifacts.ImageElement[];
       /** All the link elements on the page or equivalently declared in `Link` headers. @see https://html.spec.whatwg.org/multipage/links.html */