tests(build): use firehouse smoke test runner to test bundle (#9791)

.travis.yml

@@ -38,6 +38,7 @@ script:
   - yarn test-clients
   - yarn test-viewer
   - yarn test-lantern
+  - yarn test-bundle
   - yarn i18n:checks
   - yarn dogfood-lhci
 before_cache:

build/tests/bundle-smoke-test.js

@@ -0,0 +1,66 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs');
+const {promisify} = require('util');
+const {execFile} = require('child_process');
+const execFileAsync = promisify(execFile);
+const firehouse = require('../../lighthouse-cli/test/smokehouse/firehouse.js');
+const bundleBuilder = require('../build-bundle.js');
+const {server, serverForOffline} = require('../../lighthouse-cli/test/fixtures/static-server.js');
+
+const testEntryPath = `${__dirname}/../../lighthouse-core/index.js`;
+const testDistPath = `${__dirname}/../../dist/test-bundle.js`;
+
+/**
+ * Run Lighthouse using a CLI that shims lighthouse-core with the output of the bundler.
+ * @param {string} url
+ * @param {LH.Config.Json} config
+ */
+async function runLighthouseFromMinifiedBundle(url, config) {
+  const configPath = `${__dirname}/../../.tmp/bundle-smoke-test-config.json`;
+  const lhrPath = `${__dirname}/../../.tmp/bundle-smoke-test-lhr.json`;
+  const gatherPath = `${__dirname}/../../.tmp/bundle-smoke-test-gather`;
+
+  fs.writeFileSync(configPath, JSON.stringify(config));
+
+  await execFileAsync('node', [
+    `${__dirname}/bundled-lighthouse-cli.js`,
+    url,
+    `--config-path=${configPath}`,
+    `-GA=${gatherPath}`,
+    '--output=json',
+    `--output-path=${lhrPath}`,
+  ]);
+
+  const lhr = JSON.parse(fs.readFileSync(lhrPath, 'utf-8'));
+  const artifacts = JSON.parse(fs.readFileSync(`${gatherPath}/artifacts.json`, 'utf-8'));
+
+  return {
+    lhr,
+    artifacts,
+  };
+}
+
+async function main() {
+  await bundleBuilder.build(testEntryPath, testDistPath);
+
+  server.listen(10200, 'localhost');
+  serverForOffline.listen(10503, 'localhost');
+
+  const results = await firehouse.runSmokes({
+    runLighthouse: runLighthouseFromMinifiedBundle,
+    urlFilterRegex: /byte|dbw/,
+  });
+
+  await new Promise(resolve => server.close(resolve));
+  await new Promise(resolve => serverForOffline.close(resolve));
+
+  process.exit(results.success ? 0 : 1);
+}
+
+main();

build/tests/bundled-lighthouse-cli.js

@@ -0,0 +1,104 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview Used to smoke test the build process.
+ *
+ * - The bundled code is a function that, given a module ID, returns the exports of that module.
+ * - We eval the bundle string to get a reference to this function (with some global hacks to
+ *   support unbundleable things).
+ * - We try to locate the lighthouse-core/index.js module by executing this function on every
+ *   possible number. This version of lighthouse-core/index.js will be wired to use all of the
+ *   bundled modules, not node requires.
+ * - Once we find the bundled lighthouse-core/index.js module, we stick it in node's require.cache
+ *   so that all node require invocations for lighthouse-core/index.js will use our bundled module
+ *   instead of the regular one.
+ * - Finally, we kick off the lighthouse-cli/index.js entrypoint that ends up requiring the
+ *   now-replaced lighthouse-core/index.js for its run.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const mkdirp = require('mkdirp');
+const rimraf = require('rimraf');
+const ChromeProtocol = require('../../lighthouse-core/gather/connections/cri.js');
+
+const LH_ROOT = path.resolve(__dirname, '../..');
+const corePath = `${LH_ROOT}/lighthouse-core/index.js`;
+
+// Oh yeahhhhh this works. Think of this as `requireBundled('../../lighthouse-core/index.js')`.
+const lighthouse = (function getLighthouseCoreBundled() {
+  // The eval will assign to `require`. Normally, that would be the require on the global object.
+  // This `let` protects the global reference to the native require.
+  // Doesn't need to have any value, but for good measure define a function that explicitly forbids
+  // its own usage.
+  // To be further convinced that this works (that the un-bundled files are not being loaded),
+  // add some console.log's somewhere like `driver.js`, and
+  // run `node build/tests/bundled-lighthouse-cli.js https://www.example.com`. You won't see them.
+  /* eslint-disable-next-line */
+  let require = () => {
+    throw new Error('illegal require');
+  };
+
+  const lighthouseBundledCode = fs.readFileSync('dist/test-bundle.js', 'utf-8')
+    // Some modules are impossible to bundle. So we cheat by leaning on global.
+    // cri.js will be able to use native require. It's a minor defect - it means that some usages
+    // of lh-error.js will not come from the bundled code.
+    // TODO: use `globalThis` when we drop Node 10.
+    .replace('new ChromeProtocol', 'new global.ChromeProtocol')
+    // Needed for asset-saver.js.
+    .replace(/mkdirp\./g, 'global.mkdirp.')
+    .replace(/rimraf\./g, 'global.rimraf.')
+    .replace(/fs\.(writeFileSync|createWriteStream)/g, 'global.$&');
+
+  /* eslint-disable no-undef */
+  // @ts-ignore
+  global.ChromeProtocol = ChromeProtocol;
+  // @ts-ignore
+  global.mkdirp = mkdirp;
+  // @ts-ignore
+  global.rimraf = rimraf;
+  // @ts-ignore
+  global.fs = fs;
+  /*  eslint-enable no-undef */
+
+  const bundledLighthouseRequire = eval(lighthouseBundledCode);
+
+  // Find the lighthouse module.
+  // Every module is given an id (starting at 1). The core lighthouse module
+  // is the only module that is a function named `lighthouse`.
+  /** @type {import('../../lighthouse-core/index.js')} */
+  let lighthouse;
+  for (let i = 1; i < 1000; i++) {
+    const module = bundledLighthouseRequire(i);
+    if (module.name === 'lighthouse') {
+      lighthouse = module;
+      break;
+    }
+  }
+
+  // @ts-ignore
+  if (!lighthouse) throw new Error('could not find lighthouse module');
+
+  return lighthouse;
+})();
+
+// Shim the core module with the bundled code.
+
+// @ts-ignore
+lighthouse.__PATCHED__ = true;
+require.cache[corePath] = {
+  exports: lighthouse,
+};
+
+// @ts-ignore
+if (!require('../../lighthouse-core/index.js').__PATCHED__) {
+  throw new Error('error patching core module');
+}
+
+// Kick off the CLI.
+require('../../lighthouse-cli/index.js');

lighthouse-cli/test/smokehouse/firehouse.js

@@ -0,0 +1,68 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview Smoke test runner.
+ * Used to test channels other than npm (`run-smoke.js` handles that).
+ * Supports skipping and modifiying expectations to match the environment.
+ */
+
+/* eslint-disable no-console */
+
+const log = require('lighthouse-logger');
+const smokeTests = require('./smoke-test-dfns.js');
+const {collateResults, report} = require('./smokehouse-report.js');
+
+/**
+ * @param {Smokehouse.FirehouseOptions} options
+ */
+async function runSmokes(options) {
+  const {runLighthouse, urlFilterRegex, skip, modify} = options;
+
+  let passingCount = 0;
+  let failingCount = 0;
+
+  for (const test of smokeTests) {
+    for (const expected of test.expectations) {
+      if (urlFilterRegex && !expected.lhr.requestedUrl.match(urlFilterRegex)) {
+        continue;
+      }
+
+      console.log(`====== ${expected.lhr.requestedUrl} ======`);
+      const reasonToSkip = skip && skip(test, expected);
+      if (reasonToSkip) {
+        console.log(`skipping: ${reasonToSkip}`);
+        continue;
+      }
+
+      modify && modify(test, expected);
+      const results = await runLighthouse(expected.lhr.requestedUrl, test.config);
+      console.log(`Asserting expected results match those found. (${expected.lhr.requestedUrl})`);
+      const collated = collateResults(results, expected);
+      const counts = report(collated);
+      passingCount += counts.passed;
+      failingCount += counts.failed;
+    }
+  }
+
+  if (passingCount) {
+    console.log(log.greenify(`${passingCount} passing`));
+  }
+  if (failingCount) {
+    console.log(log.redify(`${failingCount} failing`));
+  }
+
+  return {
+    success: passingCount > 0 && failingCount === 0,
+    passingCount,
+    failingCount,
+  };
+}
+
+module.exports = {
+  runSmokes,
+};

lighthouse-core/lib/asset-saver.js

@@ -91,7 +91,7 @@ function stringifyReplacer(key, value) {
 }
 
 /**
- * Save artifacts object mostly to single file located at basePath/artifacts.log.
+ * Save artifacts object mostly to single file located at basePath/artifacts.json.
  * Also save the traces & devtoolsLogs to their own files
  * @param {LH.Artifacts} artifacts
  * @param {string} basePath

package.json

@@ -24,6 +24,7 @@
     "debug": "node --inspect-brk ./lighthouse-cli/index.js",
     "start": "node ./lighthouse-cli/index.js",
     "test": "yarn diff:sample-json && yarn lint --quiet && yarn unit && yarn type-check",
+    "test-bundle": "node build/tests/bundle-smoke-test.js",
     "test-clients": "jest \"clients/\"",
     "test-viewer": "yarn unit-viewer && jest lighthouse-viewer/test/viewer-test-pptr.js",
     "test-lantern": "bash lighthouse-core/scripts/test-lantern.sh",

types/smokehouse.d.ts

@@ -41,4 +41,11 @@
     config: LH.Config.Json;
     batch: string;
   }
+
+  export interface FirehouseOptions {
+    runLighthouse: (url: string, config: LH.Config.Json) => Promise<Omit<LH.RunnerResult, 'report'>>;
+    urlFilterRegex?: RegExp;
+    skip?: (test: TestDfn, expectation: ExpectedRunnerResult) => string | false;
+    modify?: (test: TestDfn, expectation: ExpectedRunnerResult) => void;
+  }
 }