tests: support artifact assertions in smoke tests

lighthouse-cli/test/smokehouse/a11y/expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/a11y/a11y_tester.html',
       finalUrl: 'http://localhost:10200/a11y/a11y_tester.html',
       audits: {
@@ -239,4 +240,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/byte-efficiency/tester.html',
       finalUrl: 'http://localhost:10200/byte-efficiency/tester.html',
       audits: {
@@ -117,7 +118,9 @@ module.exports = [
         },
       },
     },
+  },
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/byte-efficiency/gzip.html',
       finalUrl: 'http://localhost:10200/byte-efficiency/gzip.html',
       audits: {
@@ -142,4 +145,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
       finalUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
       audits: {
@@ -144,4 +145,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/error-expectations.js

@@ -10,15 +10,19 @@
  */
 module.exports = [
   {
+    errorCode: 'PAGE_HUNG',
+    lhr: {
       requestedUrl: 'http://localhost:10200/infinite-loop.html',
       finalUrl: 'http://localhost:10200/infinite-loop.html',
-    errorCode: 'PAGE_HUNG',
       audits: {},
     },
+  },
   {
+    errorCode: 'INSECURE_DOCUMENT_REQUEST',
+    lhr: {
       requestedUrl: 'https://expired.badssl.com',
       finalUrl: 'https://expired.badssl.com',
-    errorCode: 'INSECURE_DOCUMENT_REQUEST',
       audits: {},
     },
+  },
 ];

lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js

@@ -12,6 +12,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/online-only.html',
       finalUrl: 'http://localhost:10200/online-only.html',
       audits: {
@@ -97,8 +98,9 @@ module.exports = [
         },
       },
     },
-
+  },
   {
+    lhr: {
       requestedUrl: 'http://localhost:10503/offline-ready.html',
       finalUrl: 'http://localhost:10503/offline-ready.html',
       audits: {
@@ -158,8 +160,10 @@ module.exports = [
         },
       },
     },
+  },
 
   {
+    lhr: {
       requestedUrl: 'http://localhost:10503/offline-ready.html?slow',
       finalUrl: 'http://localhost:10503/offline-ready.html?slow',
       audits: {
@@ -171,4 +175,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/oopif-expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/oopif.html',
       finalUrl: 'http://localhost:10200/oopif.html',
       audits: {
@@ -26,4 +27,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/perf/expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/preload.html',
       finalUrl: 'http://localhost:10200/preload.html',
       audits: {
@@ -63,7 +64,9 @@ module.exports = [
         },
       },
     },
+  },
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/perf/fonts.html',
       finalUrl: 'http://localhost:10200/perf/fonts.html',
       audits: {
@@ -77,4 +80,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/perf/lantern-expectations.js

@@ -10,6 +10,7 @@
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'http://localhost:10200/online-only.html',
       finalUrl: 'http://localhost:10200/online-only.html',
       audits: {
@@ -24,4 +25,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/pwa-expectations.js

@@ -23,6 +23,7 @@ const pwaDetailsExpectations = {
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'https://airhorner.com',
       finalUrl: 'https://airhorner.com/',
       audits: {
@@ -81,8 +82,10 @@ module.exports = [
         },
       },
     },
+  },
 
   {
+    lhr: {
       requestedUrl: 'https://www.chromestatus.com/',
       finalUrl: 'https://www.chromestatus.com/features',
       audits: {
@@ -141,6 +144,7 @@ module.exports = [
         },
       },
     },
+  },
 ];
 
 module.exports.PWA_DETAILS_EXPECTATIONS = pwaDetailsExpectations;

lighthouse-cli/test/smokehouse/pwa2-expectations.js

@@ -15,6 +15,7 @@ const jakeExpectations = {...pwaDetailsExpectations, hasShortName: false};
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'https://jakearchibald.github.io/svgomg/',
       finalUrl: 'https://jakearchibald.github.io/svgomg/',
       audits: {
@@ -76,8 +77,10 @@ module.exports = [
         },
       },
     },
+  },
 
   {
+    lhr: {
       requestedUrl: 'https://shop.polymer-project.org/',
       finalUrl: 'https://shop.polymer-project.org/',
       audits: {
@@ -136,4 +139,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/pwa3-expectations.js

@@ -15,6 +15,7 @@ const pwaRocksExpectations = {...pwaDetailsExpectations, hasIconsAtLeast512px: f
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: 'https://pwa.rocks',
       finalUrl: 'https://pwa.rocks/',
       audits: {
@@ -73,4 +74,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/redirects/expectations.js

@@ -12,6 +12,7 @@ const cacheBuster = Number(new Date());
 
 module.exports = [
   {
+    lhr: {
       requestedUrl: `http://localhost:10200/online-only.html?delay=500&redirect=%2Foffline-only.html%3Fcb=${cacheBuster}%26delay=500%26redirect%3D%2Fredirects-final.html`,
       finalUrl: 'http://localhost:10200/redirects-final.html',
       audits: {
@@ -26,7 +27,9 @@ module.exports = [
         },
       },
     },
+  },
   {
+    lhr: {
       requestedUrl: `http://localhost:10200/online-only.html?delay=300&redirect=%2Fredirects-final.html`,
       finalUrl: 'http://localhost:10200/redirects-final.html',
       audits: {
@@ -41,4 +44,5 @@ module.exports = [
         },
       },
     },
+  },
 ];

lighthouse-cli/test/smokehouse/seo/expectations.js

@@ -37,6 +37,7 @@ const passHeaders = headersParam([[
  */
 module.exports = [
   {
+    lhr: {
       requestedUrl: BASE_URL + 'seo-tester.html?' + passHeaders,
       finalUrl: BASE_URL + 'seo-tester.html?' + passHeaders,
       audits: {
@@ -80,8 +81,9 @@ module.exports = [
           scoreDisplayMode: 'notApplicable',
         },
       },
-  },
+    }},
   {
+    lhr: {
       requestedUrl: BASE_URL + 'seo-failure-cases.html?' + failureHeaders,
       finalUrl: BASE_URL + 'seo-failure-cases.html?' + failureHeaders,
       audits: {
@@ -141,7 +143,9 @@ module.exports = [
         },
       },
     },
+  },
   {
+    lhr: {
       // Note: most scores are null (audit error) because the page 403ed.
       requestedUrl: BASE_URL + 'seo-failure-cases.html?status_code=403',
       finalUrl: BASE_URL + 'seo-failure-cases.html?status_code=403',
@@ -178,10 +182,11 @@ module.exports = [
           score: null,
         },
       },
-  },
+    }},
   {
-    requestedUrl: BASE_URL + 'seo-tap-targets.html',
+    lhr: {
       finalUrl: BASE_URL + 'seo-tap-targets.html',
+      requestedUrl: BASE_URL + 'seo-tap-targets.html',
       audits: {
         'tap-targets': {
           score: (() => {
@@ -221,4 +226,10 @@ module.exports = [
         },
       },
     },
+    artifacts: {
+      TapTargets: {
+        length: 11,
+      },
+    },
+  },
 ];

lighthouse-cli/test/smokehouse/smoke-test-dfns.js

@@ -110,7 +110,7 @@ function loadConfig(configPath) {
 
 /**
  * @param {string} expectationsPath
- * @return {Smokehouse.ExpectedLHR[]}
+ * @return {Smokehouse.ExpectedRunnerResult[]}
  */
 function loadExpectations(expectationsPath) {
   return require(expectationsPath);