tests(build): use firehouse smoke test runner to test bundle

.travis.yml

@@ -38,6 +38,7 @@ script:
   - yarn test-clients
   - yarn test-viewer
   - yarn test-lantern
+  - yarn test-bundle
   - yarn i18n:checks
   - yarn dogfood-lhci
 before_cache:

build/tests/bundle-smoke-test.js

@@ -0,0 +1,66 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs');
+const {promisify} = require('util');
+const {execFile} = require('child_process');
+const execFileAsync = promisify(execFile);
+const firehouse = require('../../lighthouse-cli/test/smokehouse/firehouse.js');
+const bundleBuilder = require('../build-bundle.js');
+const {server, serverForOffline} = require('../../lighthouse-cli/test/fixtures/static-server.js');
+
+const testEntryPath = `${__dirname}/../../lighthouse-core/index.js`;
+const testDistPath = `${__dirname}/../../dist/test-bundle.js`;
+
+/**
+ * Run Lighthouse using a CLI that shims lighthouse-core with the output of the bundler.
+ * @param {string} url
+ * @param {LH.Config.Json} config
+ */
+async function runLighthouseFromMinifiedBundle(url, config) {
+  const configPath = `${__dirname}/../../.tmp/bundle-smoke-test-config.json`;
+  const lhrPath = `${__dirname}/../../.tmp/bundle-smoke-test-lhr.json`;
+  const gatherPath = `${__dirname}/../../.tmp/bundle-smoke-test-gather`;
+
+  fs.writeFileSync(configPath, JSON.stringify(config));
+
+  await execFileAsync('node', [
+    `${__dirname}/bundled-lighthouse-cli.js`,
+    url,
+    `--config-path=${configPath}`,
+    `-GA=${gatherPath}`,
+    '--output=json',
+    `--output-path=${lhrPath}`,
+  ]);
+
+  const lhr = JSON.parse(fs.readFileSync(lhrPath, 'utf-8'));
+  const artifacts = JSON.parse(fs.readFileSync(`${gatherPath}/artifacts.json`, 'utf-8'));
+
+  return {
+    lhr,
+    artifacts,
+  };
+}
+
+async function main() {
+  await bundleBuilder.build(testEntryPath, testDistPath);
+
+  server.listen(10200, 'localhost');
+  serverForOffline.listen(10503, 'localhost');
+
+  const results = await firehouse.runSmokes({
+    runLighthouse: runLighthouseFromMinifiedBundle,
+    filter: /byte|dbw/,
+  });
+
+  await new Promise(resolve => server.close(resolve));
+  await new Promise(resolve => serverForOffline.close(resolve));
+
+  process.exit(results.success ? 0 : 1);
+}
+
+main();

build/tests/bundled-lighthouse-cli.js

@@ -0,0 +1,88 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const mkdirp = require('mkdirp');
+const rimraf = require('rimraf');
+const ChromeProtocol = require('../../lighthouse-core/gather/connections/cri.js');
+
+const LH_ROOT = path.resolve(__dirname, '../..');
+const corePath = `${LH_ROOT}/lighthouse-core/index.js`;
+
+// Oh yeahhhhh this works. Think of this as `requireBundled('../../lighthouse-core/index.js')`.
+const lighthouse = (function getLighthouseCoreBundled() {
+  // The eval will assign to `require`. Normally, that would be the require on the global object.
+  // This `let` protects the global reference to the native require.
+  // Doesn't need to have any value, but for good measure define a function that explicitly forbids
+  // its own usage.
+  // To be further convinced that this works (that the un-bundled files are not being loaded),
+  // add some console.log's somewhere like `driver.js`, and
+  // run `node build/tests/bundled-lighthouse-cli.js https://www.example.com`. You won't see them.
+  /* eslint-disable-next-line */
+  let require = () => {
+    throw new Error('illegal require');
+  };
+
+  const lighthouseBundledCode = fs.readFileSync('dist/test-bundle.js', 'utf-8')
+    // Some modules are impossible to bundle. So we cheat by leaning on global.
+    // cri.js will be able to use native require. It's a minor defect - it means that some usages
+    // of lh-error.js will not come from the bundled code.
+    // TODO: use `globalThis` when we drop Node 10.
+    .replace('new ChromeProtocol', 'new global.ChromeProtocol')
+    // Needed for asset-saver.js.
+    .replace(/mkdirp\./g, 'global.mkdirp.')
+    .replace(/rimraf\./g, 'global.rimraf.')
+    .replace(/fs\.(writeFileSync|createWriteStream)/g, 'global.$&');
+
+  /* eslint-disable no-undef */
+  // @ts-ignore
+  global.ChromeProtocol = ChromeProtocol;
+  // @ts-ignore
+  global.mkdirp = mkdirp;
+  // @ts-ignore
+  global.rimraf = rimraf;
+  // @ts-ignore
+  global.fs = fs;
+  /*  eslint-enable no-undef */
+
+  const bundledLighthouseRequire = eval(lighthouseBundledCode);
+
+  // Find the lighthouse module.
+  // Every module is given an id (starting at 1). The core lighthouse module
+  // is the only module that is a function named `lighthouse`.
+  /** @type {import('../../lighthouse-core/index.js')} */
+  let lighthouse;
+  for (let i = 1; i < 1000; i++) {
+    const module = bundledLighthouseRequire(i);
+    if (module.name === 'lighthouse') {
+      lighthouse = module;
+      break;
+    }
+  }
+
+  // @ts-ignore
+  if (!lighthouse) throw new Error('could not find lighthouse module');
+
+  return lighthouse;
+})();
+
+// Shim the core module with the bundled code.
+
+// @ts-ignore
+lighthouse.__PATCHED__ = true;
+require.cache[corePath] = {
+  exports: lighthouse,
+};
+
+// @ts-ignore
+if (!require('../../lighthouse-core/index.js').__PATCHED__) {
+  throw new Error('error patching core module');
+}
+
+// Kick off the CLI.
+require('../../lighthouse-cli/index.js');

lighthouse-cli/test/smokehouse/firehouse.js

@@ -0,0 +1,68 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview Smoke test runner.
+ * Used to test channels other than npm (`run-smoke.js` handles that).
+ * Supports skipping and modifiying expectations to match the environment.
+ */
+
+/* eslint-disable no-console */
+
+const log = require('lighthouse-logger');
+const smokeTests = require('./smoke-test-dfns.js');
+const {collateResults, report} = require('./smokehouse-report.js');
+
+/**
+ * @param {Smokehouse.FirehouseOptions} options
+ */
+async function runSmokes(options) {
+  const {runLighthouse, filter, skip, modify} = options;
+
+  let passingCount = 0;
+  let failingCount = 0;
+
+  for (const test of smokeTests) {
+    for (const expected of test.expectations) {
+      if (filter && !expected.lhr.requestedUrl.match(filter)) {
+        continue;
+      }
+
+      console.log(`====== ${expected.lhr.requestedUrl} ======`);
+      const reasonToSkip = skip && skip(test, expected);
+      if (reasonToSkip) {
+        console.log(`skipping: ${reasonToSkip}`);
+        continue;
+      }
+
+      modify && modify(test, expected);
+      const results = await runLighthouse(expected.lhr.requestedUrl, test.config);
+      console.log(`Asserting expected results match those found. (${expected.lhr.requestedUrl})`);
+      const collated = collateResults(results, expected);
+      const counts = report(collated);
+      passingCount += counts.passed;
+      failingCount += counts.failed;
+    }
+  }
+
+  if (passingCount) {
+    console.log(log.greenify(`${passingCount} passing`));
+  }
+  if (failingCount) {
+    console.log(log.redify(`${failingCount} failing`));
+  }
+
+  return {
+    success: passingCount > 0 && failingCount === 0,
+    passingCount,
+    failingCount,
+  };
+}
+
+module.exports = {
+  runSmokes,
+};

lighthouse-core/lib/asset-saver.js

@@ -91,7 +91,7 @@ function stringifyReplacer(key, value) {
 }
 
 /**
- * Save artifacts object mostly to single file located at basePath/artifacts.log.
+ * Save artifacts object mostly to single file located at basePath/artifacts.json.
  * Also save the traces & devtoolsLogs to their own files
  * @param {LH.Artifacts} artifacts
  * @param {string} basePath

package.json

@@ -24,6 +24,7 @@
     "debug": "node --inspect-brk ./lighthouse-cli/index.js",
     "start": "node ./lighthouse-cli/index.js",
     "test": "yarn diff:sample-json && yarn lint --quiet && yarn unit && yarn type-check",
+    "test-bundle": "node build/tests/bundle-smoke-test.js",
     "test-clients": "jest \"clients/\"",
     "test-viewer": "yarn unit-viewer && jest lighthouse-viewer/test/viewer-test-pptr.js",
     "test-lantern": "bash lighthouse-core/scripts/test-lantern.sh",

types/smokehouse.d.ts

@@ -41,4 +41,11 @@
     config: LH.Config.Json;
     batch: string;
   }
+
+  export interface FirehouseOptions {
+    runLighthouse: (url: string, config: LH.Config.Json) => Promise<Omit<LH.RunnerResult, 'report'>>;
+    filter?: RegExp;
+    skip?: (test: TestDfn, expectation: ExpectedRunnerResult) => string | false;
+    modify?: (test: TestDfn, expectation: ExpectedRunnerResult) => void;
+  }
 }