Skip to content

Commit

Permalink
tests(smoke): support full chrome version for pruning (#13896)
Browse files Browse the repository at this point in the history
  • Loading branch information
connorjclark authored Apr 22, 2022
1 parent 1daf915 commit 2a8058a
Show file tree
Hide file tree
Showing 14 changed files with 127 additions and 23 deletions.
9 changes: 5 additions & 4 deletions lighthouse-cli/test/smokehouse/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,15 +82,16 @@ Arrays can be asserted to not match any elements using the special `_excludes` p

### Special environment checks

If an expectation requires a minimum version of Chromium, use `_minChromiumMilestone: xx` to conditionally ignore that entire object in the expectation.
If an expectation requires a minimum version of Chromium, use `_minChromiumVersion: xx.x.x.x` to conditionally ignore that entire object in the expectation.
Can be as specific as you like (`_minChromiumVersion: xx` works too).

**Examples**:
```js
{
artifacts: {
InspectorIssues: {
// Mixed Content issues weren't added to the protocol until M84.
_minChromiumMilestone: 84, // The entire `InspectorIssues` is ignored for older Chrome.
_minChromiumVersion: '84', // The entire `InspectorIssues` is ignored for older Chrome.
mixedContent: [
{
resourceType: 'Image',
Expand All @@ -111,8 +112,8 @@ If an expectation requires a minimum version of Chromium, use `_minChromiumMiles
All pruning checks:
- `_minChromiumMilestone`
- `_maxChromiumMilestone`
- `_minChromiumVersion`
- `_maxChromiumVersion`
- `_legacyOnly`
- `_fraggleRockOnly`
- `_skipInBundled`
Expand Down
25 changes: 15 additions & 10 deletions lighthouse-cli/test/smokehouse/report-assert.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import _ from 'lodash';
import log from 'lighthouse-logger';

import {LocalConsole} from './lib/local-console.js';
import {chromiumVersionCheck} from './version-check.js';

const {cloneDeep} = _;

Expand Down Expand Up @@ -227,22 +228,26 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {

/**
* Lazily compute the Chrome version because some reports are explicitly asserting error conditions.
* @returns {number}
* @returns {string}
*/
function getChromeVersion() {
function getChromeVersionString() {
const userAgent = lhr.environment.hostUserAgent;
const userAgentMatch = /Chrome\/(\d+)/.exec(userAgent); // Chrome/85.0.4174.0
const userAgentMatch = /Chrome\/([\d.]+)/.exec(userAgent); // Chrome/85.0.4174.0
if (!userAgentMatch) throw new Error('Could not get chrome version.');
return Number(userAgentMatch[1]);
const versionString = userAgentMatch[1];
if (versionString.split('.').length !== 4) throw new Error(`unexpected ua: ${userAgent}`);
return versionString;
}

/**
* @param {*} obj
*/
function failsChromeVersionCheck(obj) {
if (obj._minChromiumMilestone && getChromeVersion() < obj._minChromiumMilestone) return true;
if (obj._maxChromiumMilestone && getChromeVersion() > obj._maxChromiumMilestone) return true;
return false;
return !chromiumVersionCheck({
version: getChromeVersionString(),
min: obj._minChromiumVersion,
max: obj._maxChromiumVersion,
});
}

/**
Expand Down Expand Up @@ -271,7 +276,7 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
localConsole.log([
`[${key}] failed chrome version check, pruning expectation:`,
JSON.stringify(value, null, 2),
`Actual Chromium version: ${getChromeVersion()}`,
`Actual Chromium version: ${getChromeVersionString()}`,
].join(' '));
remove(key);
} else if (value._legacyOnly && isFraggleRock) {
Expand Down Expand Up @@ -307,8 +312,8 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
delete obj._legacyOnly;
delete obj._fraggleRockOnly;
delete obj._skipInBundled;
delete obj._minChromiumMilestone;
delete obj._maxChromiumMilestone;
delete obj._minChromiumVersion;
delete obj._maxChromiumVersion;
delete obj._runner;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ const expectations = {
'unused-javascript': {
// ScriptParsedEvent.embedderName wasn't added to the protocol until M86.
// https://chromiumdash.appspot.com/commit/52ed57138d0b83e8afd9de25e60655c6ace7527c
_minChromiumMilestone: 86,
_minChromiumVersion: '86',
score: '<1',
details: {
// the specific ms value here is not meaningful for this smoketest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const blockAllExceptInlineScriptCsp = headersParam([[
*/
const expectations = {
artifacts: {
_minChromiumMilestone: 92,
_minChromiumVersion: '92',
RobotsTxt: {
status: 200,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ const expectations = {
finalUrl: 'http://localhost:10200/dobetterweb/dbw_tester.html',
audits: {
'errors-in-console': {
_minChromiumMilestone: 95,
_minChromiumVersion: '95',
score: 0,
details: {
items: {
Expand Down Expand Up @@ -321,6 +321,7 @@ const expectations = {
},
},
{
_maxChromiumVersion: '103.0.5017.0',
value: /Synchronous XMLHttpRequest on the main thread is deprecated/,
source: {
type: 'source-location',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const expectations = {
InspectorIssues: {
mixedContentIssue: [
{
_minChromiumMilestone: 88, // We went from Warning to AutoUpgrade in https://chromium-review.googlesource.com/c/chromium/src/+/2480817
_minChromiumVersion: '88', // We went from Warning to AutoUpgrade in https://chromium-review.googlesource.com/c/chromium/src/+/2480817
resourceType: 'Image',
resolutionStatus: 'MixedContentAutomaticallyUpgraded',
insecureURL: 'http://www.mixedcontentexamples.com/Content/Test/steveholt.jpg',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ const expectations = {
artifacts: {
InstallabilityErrors: {
// COMPAT: `warn-not-offline-capable` occurs in m89 but may be cherry-picked out of m90.
_minChromiumMilestone: 91,
_minChromiumVersion: '91',
errors: {
length: 1,
// 0: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const expectations = {
'non-composited-animations': {
// Requires compositor failure reasons to be in the trace
// https://chromiumdash.appspot.com/commit/995baabedf9e70d16deafc4bc37a2b215a9b8ec9
_minChromiumMilestone: 86,
_minChromiumVersion: '86',
score: null,
displayValue: '1 animated element found',
details: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ const expectations = {
//
// Weighted score on emulated mobile bug fixed in m92:
// https://chromium.googlesource.com/chromium/src/+/042fbfb4cc6a675da0dff4bf3fc08622af42422b
_minChromiumMilestone: 92,
_minChromiumVersion: '92',
firstContentfulPaint: '>5000',
firstContentfulPaintAllFrames: '<5000',
largestContentfulPaint: '>5000',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ const expectations = {
// Requires compositor failure reasons to be in the trace
// for `failureReasonsMask` and `unsupportedProperties`
// https://chromiumdash.appspot.com/commit/995baabedf9e70d16deafc4bc37a2b215a9b8ec9
_minChromiumMilestone: 86,
_minChromiumVersion: '86',
traceEventType: 'animation',
node: {
selector: 'body > div#animate-me',
Expand Down
45 changes: 45 additions & 0 deletions lighthouse-cli/test/smokehouse/version-check-test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/**
* @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
*/

import {chromiumVersionCheck, compareVersions} from './version-check.js';

/* eslint-env jest */

describe('version check', () => {
it('compareVersions', async () => {
expect(compareVersions([100, 0, 0, 0], [100, 0, 0, 0])).toBe(0);
expect(compareVersions([101, 0, 0, 0], [100, 0, 0, 0])).toBe(1);
expect(compareVersions([99, 0, 0, 0], [100, 0, 0, 0])).toBe(-1);

expect(compareVersions([100, 0, 10, 0], [100, 0, 10, 0])).toBe(0);
expect(compareVersions([100, 0, 11, 0], [100, 0, 10, 0])).toBe(1);
expect(compareVersions([100, 0, 9, 0], [100, 0, 10, 0])).toBe(-1);

expect(compareVersions([100, 0, 0, 0], [100])).toBe(0);
expect(compareVersions([100, 0, 0, 1], [100])).toBe(1);
expect(compareVersions([99, 0, 0, 0], [100])).toBe(-1);
});

it('chromiumVersionCheck', async () => {
expect(chromiumVersionCheck({version: '100'})).toBe(true);
expect(chromiumVersionCheck({version: '100', min: '100'})).toBe(true);
expect(chromiumVersionCheck({version: '100', max: '100'})).toBe(true);
expect(chromiumVersionCheck({version: '100', min: '101'})).toBe(false);
expect(chromiumVersionCheck({version: '100', max: '99'})).toBe(false);

expect(chromiumVersionCheck({version: '100.0.2331.3'})).toBe(true);
expect(chromiumVersionCheck({version: '100.0.2331.3', min: '100.0.2331.3'})).toBe(true);
expect(chromiumVersionCheck({version: '100.0.2331.3', min: '100.0.0.0'})).toBe(true);
expect(chromiumVersionCheck({version: '100.0.2331.3', max: '100.0.3333.3'})).toBe(true);
expect(chromiumVersionCheck({version: '100.0.2331.3', min: '100.0.2331.2'})).toBe(true);
expect(chromiumVersionCheck({version: '100.0.2331.3', max: '99'})).toBe(false);

expect(chromiumVersionCheck({
version: '100.0.2331.3', min: '100.0.2331.0', max: '100.0.2331.10'})).toBe(true);
expect(chromiumVersionCheck({
version: '100.3.2331.3', min: '100.0.2331.0', max: '100.0.2331.10'})).toBe(false);
});
});
48 changes: 48 additions & 0 deletions lighthouse-cli/test/smokehouse/version-check.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/**
* @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
*/

/**
* @fileoverview Compares chromium version strings: 103.0.5017.0
*/

/**
* @param {string} versionString
* @return {number[]}
*/
function parseVersion(versionString) {
const versionParts = versionString.split('.');
return versionParts.map(Number);
}

/**
* @param {number[]} versionA
* @param {number[]} versionB
*/
function compareVersions(versionA, versionB) {
for (let i = 0; i < versionA.length; i++) {
if ((versionA[i] ?? 0) > (versionB[i] ?? 0)) return 1;
if ((versionA[i] ?? 0) < (versionB[i] ?? 0)) return -1;
}
return 0;
}

/**
* Returns false if fails check.
* @param {{version: string, min?: string, max?: string}} opts
*/
function chromiumVersionCheck(opts) {
const version = parseVersion(opts.version);
const min = opts.min && parseVersion(opts.min);
const max = opts.max && parseVersion(opts.max);
if (min && compareVersions(version, min) === -1) return false;
if (max && compareVersions(version, max) === 1) return false;
return true;
}

export {
chromiumVersionCheck,
compareVersions,
};
4 changes: 4 additions & 0 deletions lighthouse-core/audits/deprecations.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ class Deprecations extends Audit {
let deprecations;
if (artifacts.InspectorIssues.deprecationIssue.length) {
deprecations = artifacts.InspectorIssues.deprecationIssue
// TODO: translate these strings.
// see https://github.com/GoogleChrome/lighthouse/issues/13895
// @ts-expect-error: .type hasn't released to npm yet
.filter(deprecation => !deprecation.type || deprecation.type === 'Untranslated')
.map(deprecation => {
const {scriptId, url, lineNumber, columnNumber} = deprecation.sourceCodeLocation;
const bundle = bundles.find(bundle => bundle.script.scriptId === scriptId);
Expand Down
2 changes: 1 addition & 1 deletion types/smokehouse.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ declare global {

export type ExpectedRunnerResult = {
lhr: ExpectedLHR,
artifacts?: Partial<Record<keyof Artifacts|'_maxChromiumMilestone'|'_minChromiumMilestone', any>>
artifacts?: Partial<Record<keyof Artifacts|'_maxChromiumVersion'|'_minChromiumVersion', any>>
networkRequests?: {length: number, _legacyOnly?: boolean, _fraggleRockOnly?: boolean};
}

Expand Down

0 comments on commit 2a8058a

Please sign in to comment.