From f5a478cd9ec4b20e739579d1d097765c7afce61f Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Wed, 18 Apr 2018 09:22:34 -0700 Subject: [PATCH] tests(smokehouse): split pwa2, perf batches --- .../test/smokehouse/pwa2-expectations.js | 105 ---------------- .../test/smokehouse/pwa3-expectations.js | 117 ++++++++++++++++++ lighthouse-cli/test/smokehouse/run-smoke.js | 8 +- 3 files changed, 123 insertions(+), 107 deletions(-) create mode 100644 lighthouse-cli/test/smokehouse/pwa3-expectations.js diff --git a/lighthouse-cli/test/smokehouse/pwa2-expectations.js b/lighthouse-cli/test/smokehouse/pwa2-expectations.js index 057200f06b95..d482f64a0e64 100644 --- a/lighthouse-cli/test/smokehouse/pwa2-expectations.js +++ b/lighthouse-cli/test/smokehouse/pwa2-expectations.js @@ -227,109 +227,4 @@ module.exports = [ }, }, }, - - { - initialUrl: 'https://pwa.rocks', - url: 'https://pwa.rocks/', - audits: { - 'is-on-https': { - score: 1, - }, - 'redirects-http': { - score: 1, - }, - 'service-worker': { - score: 1, - }, - 'works-offline': { - score: 1, - }, - 'viewport': { - score: 1, - }, - 'without-javascript': { - score: 1, - }, - 'load-fast-enough-for-pwa': { - // Ignore speed test; just verify that it ran . - }, - 'webapp-install-banner': { - score: 1, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'splash-screen': { - score: 0, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'themed-omnibox': { - score: 0, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'content-width': { - score: 1, - }, - - // "manual" audits. Just verify in the results. - 'pwa-cross-browser': { - score: 0, - manual: true, - }, - 'pwa-page-transitions': { - score: 0, - manual: true, - }, - 'pwa-each-page-has-url': { - score: 0, - manual: true, - }, - }, - }, ]; diff --git a/lighthouse-cli/test/smokehouse/pwa3-expectations.js b/lighthouse-cli/test/smokehouse/pwa3-expectations.js new file mode 100644 index 000000000000..ad7de011d2fa --- /dev/null +++ b/lighthouse-cli/test/smokehouse/pwa3-expectations.js @@ -0,0 +1,117 @@ +/** + * @license Copyright 2016 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Expected Lighthouse audit values for various sites with stable(ish) PWA + * results. + */ +module.exports = [ + { + initialUrl: 'https://pwa.rocks', + url: 'https://pwa.rocks/', + audits: { + 'is-on-https': { + score: 1, + }, + 'redirects-http': { + score: 1, + }, + 'service-worker': { + score: 1, + }, + 'works-offline': { + score: 1, + }, + 'viewport': { + score: 1, + }, + 'without-javascript': { + score: 1, + }, + 'load-fast-enough-for-pwa': { + // Ignore speed test; just verify that it ran . + }, + 'webapp-install-banner': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'splash-screen': { + score: 0, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'themed-omnibox': { + score: 0, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'content-width': { + score: 1, + }, + + // "manual" audits. Just verify in the results. + 'pwa-cross-browser': { + score: 0, + manual: true, + }, + 'pwa-page-transitions': { + score: 0, + manual: true, + }, + 'pwa-each-page-has-url': { + score: 0, + manual: true, + }, + }, + }, +]; diff --git a/lighthouse-cli/test/smokehouse/run-smoke.js b/lighthouse-cli/test/smokehouse/run-smoke.js index c91cacd8ef3d..86639f9c5918 100644 --- a/lighthouse-cli/test/smokehouse/run-smoke.js +++ b/lighthouse-cli/test/smokehouse/run-smoke.js @@ -36,6 +36,10 @@ const SMOKETESTS = [{ id: 'pwa2', expectations: smokehouseDir + 'pwa2-expectations.js', config: smokehouseDir + 'pwa-config.js', +}, { + id: 'pwa3', + expectations: smokehouseDir + 'pwa3-expectations.js', + config: smokehouseDir + 'pwa-config.js', }, { id: 'dbw', expectations: 'dobetterweb/dbw-expectations.js', @@ -56,12 +60,12 @@ const SMOKETESTS = [{ id: 'byte', expectations: 'byte-efficiency/expectations.js', config: smokehouseDir + 'byte-config.js', - batch: 'perf', + batch: 'perf-opportunity', }, { id: 'perf', expectations: 'perf/expectations.js', config: 'lighthouse-core/config/perf-config.js', - batch: 'perf', + batch: 'perf-metric', }, { id: 'ttci', expectations: 'tricky-ttci/expectations.js',