tests: use smokehouse runner for test-bundle (#9943)

.travis.yml

@@ -38,7 +38,7 @@ script:
   - yarn test-clients
   - yarn test-viewer
   - yarn test-lantern
-  - yarn test-bundle || yarn test-bundle || yarn test-bundle
+  - yarn test-bundle
   - yarn i18n:checks
   - yarn dogfood-lhci
   - yarn test-docs

build/build-bundle.js

@@ -38,7 +38,8 @@ const isDevtools = file => path.basename(file).includes('devtools');
 /** @param {string} file */
 const isLightrider = file => path.basename(file).includes('lightrider');
 
-const BANNER = `// lighthouse, browserified. ${VERSION} (${COMMIT_HASH})\n`;
+const BANNER = `// lighthouse, browserified. ${VERSION} (${COMMIT_HASH})\n` +
+  '// @ts-nocheck\n'; // To prevent tsc stepping into any required bundles.
 const DEBUG = false; // true for sourcemaps
 
 /**

clients/devtools-entry.js

@@ -40,16 +40,14 @@ function listenForStatus(listenCallback) {
   log.events.addListener('status', listenCallback);
 }
 
+// For the bundle smoke test.
 if (typeof module !== 'undefined' && module.exports) {
-  // export for require()ing (via browserify).
-  module.exports = {
-    setUpWorkerConnection,
-    lighthouse,
-    getDefaultConfigForCategories,
-    listenForStatus,
-    registerLocaleData,
-    lookupLocale,
-  };
+  // Ideally this could be exposed via browserify's `standalone`, but it doesn't
+  // work for LH because of https://github.com/browserify/browserify/issues/968
+  // Instead, since this file is only ever run in node for testing, expose a
+  // bundle entry point as global.
+  // @ts-ignore
+  global.runBundledLighthouse = lighthouse;
 }
 
 // Expose only in DevTools' worker

lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -19,12 +19,15 @@ const log = require('lighthouse-logger');
 const {runSmokehouse} = require('../smokehouse.js');
 const {server, serverForOffline} = require('../../fixtures/static-server.js');
 
-const cliLighthouseRunner = require('../lighthouse-runners/cli.js').runLighthouse;
-
 const coreTestDefnsPath = require.resolve('../test-definitions/core-tests.js');
 
-const runners = {
-  cli: cliLighthouseRunner,
+/**
+ * Possible Lighthouse runners. Loaded dynamically so e.g. a CLI run isn't
+ * contingent on having built all the bundles.
+ */
+const runnerPaths = {
+  cli: '../lighthouse-runners/cli.js',
+  bundle: '../lighthouse-runners/bundle.js',
 };
 
 /**
@@ -76,7 +79,7 @@ async function begin() {
     .alias({
       'jobs': 'j',
     })
-    .choices('runner', ['cli'])
+    .choices('runner', ['cli', 'bundle'])
     .default('runner', 'cli')
     .wrap(yargs.terminalWidth())
     .argv;
@@ -85,7 +88,11 @@ async function begin() {
   const jobs = argv.jobs !== undefined ? Number(argv.jobs) : undefined;
   const retries = argv.retries !== undefined ? Number(argv.retries) : undefined;
 
-  const lighthouseRunner = runners[/** @type {keyof typeof runners} */ (argv.runner)];
+  const runnerPath = runnerPaths[/** @type {keyof typeof runnerPaths} */ (argv.runner)];
+  if (argv.runner === 'bundle') {
+    console.log('\n✨ Be sure to have recently run this: yarn build-all');
+  }
+  const lighthouseRunner = require(runnerPath).runLighthouse;
 
   // Find test definition file and filter by requestedTestIds.
   let testDefnPath = argv.testsPath || coreTestDefnsPath;

lighthouse-cli/test/smokehouse/lighthouse-runners/bundle.js

@@ -0,0 +1,53 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview  A runner that launches Chrome and executes Lighthouse via a
+ * bundle to test that bundling has produced correct and runnable code.
+ * Currently uses `lighthouse-dt-bundle.js`.
+ */
+
+const ChromeLauncher = require('chrome-launcher');
+const ChromeProtocol = require('../../../../lighthouse-core/gather/connections/cri.js');
+
+// Load bundle, which creates a `global.runBundledLighthouse`.
+require('../../../../dist/lighthouse-dt-bundle.js');
+
+/** @type {import('../../../../lighthouse-core/index.js')} */
+// @ts-ignore - not worth giving test global an actual type.
+const lighthouse = global.runBundledLighthouse;
+
+/**
+ * Launch Chrome and do a full Lighthouse run via the Lighthouse CLI.
+ * @param {string} url
+ * @param {LH.Config.Json=} configJson
+ * @param {{isDebug?: boolean}=} testRunnerOptions
+ * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
+ */
+async function runLighthouse(url, configJson, testRunnerOptions = {}) {
+  // Launch and connect to Chrome.
+  const launchedChrome = await ChromeLauncher.launch();
+  const port = launchedChrome.port;
+  const connection = new ChromeProtocol(port);
+
+  // Run Lighthouse.
+  const logLevel = testRunnerOptions.isDebug ? 'info' : undefined;
+  const runnerResult = await lighthouse(url, {port, logLevel}, configJson, connection);
+  if (!runnerResult) throw new Error('No runnerResult');
+
+  // Clean up and return results.
+  await launchedChrome.kill();
+  return {
+    lhr: runnerResult.lhr,
+    artifacts: runnerResult.artifacts,
+    log: '', // TODO: if want to run in parallel, need to capture lighthouse-logger output.
+  };
+}
+
+module.exports = {
+  runLighthouse,
+};

lighthouse-cli/test/smokehouse/readme.md

@@ -79,20 +79,20 @@ The different frontends launch smokehouse with a set of tests to run. Smokehouse
 Smokehouse Frontends                                        Lighthouse Runners
 +------------+
 |            |
-|  bin (CLI) +----+                                           +--------------+
+|   bin.js   +----+                                           +--------------+
 |            |    |                                           |              |
-+------------+    |                                       +-->+     cli      |
++------------+    |                                       +-->+    cli.js    |
                   |                                       |   |              |
 +------------+    |            +---------------+          |   +--------------+
 |            |    | testDefns> |               |  config> |
-|   node     +---------------->+ smokehouse.js +<---------+
+|   node.js  +---------------->+ smokehouse.js +<---------+
 |            |    |            |               |   <lhr   |   +--------------+
 +------------+    |            +-------+-------+          |   |              |
-                  |                    ^                  +-->+   bundled    |
+                  |                    ^                  +-->+   bundle.js  |
 +------------+    |                    |                      |              |
 |            |    |                    |                      +--------------+
 |bundle+entry+----+                    v
-|            |                +--------+--------+
+|   (TODO)   |                +--------+--------+
 +------------+                |                 |
                               |  report/assert  |
                               |                 |
@@ -111,8 +111,8 @@ Smokehouse Frontends                                        Lighthouse Runners
 
 ### Lighthouse runners
 
-- `runners/cli.js` - the original test runner, exercising the Lighthouse CLI from command-line argument parsing to the results written to disk on completion.
-- TODO: bundle runner - a smoke test runner that operates on an already-bundled version of Lighthouse for end-to-end testing of that version.
+- `lighthouse-runners/cli.js` - the original test runner, exercising the Lighthouse CLI from command-line argument parsing to the results written to disk on completion.
+- `lighthouse-runners/bundle.js` - a smoke test runner that operates on an already-bundled version of Lighthouse for end-to-end testing of that version.
 
 ## Custom smoke tests (for plugins et al.)
 

package.json

@@ -24,7 +24,7 @@
     "debug": "node --inspect-brk ./lighthouse-cli/index.js",
     "start": "node ./lighthouse-cli/index.js",
     "test": "yarn diff:sample-json && yarn lint --quiet && yarn unit && yarn type-check",
-    "test-bundle": "node build/tests/bundle-smoke-test.js",
+    "test-bundle": "yarn smoke --runner bundle -j=1 --retries=2 dbw byte",
     "test-clients": "jest \"clients/\"",
     "test-viewer": "yarn unit-viewer && jest lighthouse-viewer/test/viewer-test-pptr.js",
     "test-lantern": "bash lighthouse-core/scripts/test-lantern.sh",