clients: reorganize file structure and build process (#6344)

.appveyor.yml

@@ -48,6 +48,5 @@ test_script:
 cache:
   #- chrome-win32 -> appveyor.yml,package.json
   - node_modules -> appveyor.yml,package.json,yarn.lock
-  - lighthouse-extension\node_modules -> appveyor.yml,package.json,yarn.lock,lighthouse-extension\package.json,lighthouse-extension\yarn.lock
   - lighthouse-viewer\node_modules -> appveyor.yml,package.json,yarn.lock,lighthouse-viewer\package.json,lighthouse-viewer\yarn.lock
   - '%LOCALAPPDATA%\Yarn -> appveyor.yml,package.json,yarn.lock'

.cz-config.js

@@ -18,7 +18,7 @@ module.exports = {
     {value: 'deps',       name: 'deps:      Dependency bumps only'},
     {value: 'report',     name: 'report:    Report, UI, renderers'},
     {value: 'cli',        name: 'cli:       CLI stuff'},
-    {value: 'extension',  name: 'extension: Chrome extension stuff'},
+    {value: 'clients',    name: 'clients:   Extension, DevTools, or LR stuff'},
     {value: 'misc',       name: 'misc:      Something else entirely'}
   ]
 };

.eslintignore

@@ -6,9 +6,7 @@
 **/third_party/**
 
 **/dist/**
-lighthouse-extension/app/scripts
 
-**/closure/*/*
 coverage/**
 
 !.eslintrc.js

.github/CODEOWNERS

@@ -12,7 +12,7 @@ lighthouse-core/report/             @paulirish
 
 lighthouse-core/config/             @patrickhulce       @brendankenny
 lighthouse-core/lib/                @brendankenny       @paulirish
-lighthouse-extension/               @paulirish          @patrickhulce
+clients/                            @paulirish          @patrickhulce
 lighthouse-viewer/                  @ebidel             @brendankenny
 
 docs/                               @paulirish

.gitignore

@@ -4,9 +4,7 @@ npm-debug.log
 .vscode
 .tmp
 
-lighthouse-extension/dist
-lighthouse-core/third_party/src
-**/pages/scripts/lighthouse-report.js
+dist
 coverage
 lcov.info
 .nyc_output

.npmignore

@@ -11,7 +11,7 @@ lighthouse-core/scripts/*
 lighthouse-core/test/
 lighthouse-core/third_party/src/
 lighthouse-viewer/
-lighthouse-extension/
+clients/
 lighthouse-cli/results/
 lighthouse-logger/
 

.travis.yml

@@ -14,7 +14,6 @@ cache:
   directories:
     - node_modules
     - lantern-data
-    - lighthouse-extension/node_modules
     - lighthouse-viewer/node_modules
     - /home/travis/.rvm/gems/
 install:
@@ -38,7 +37,7 @@ script:
   - yarn unit:silentcoverage
   - yarn type-check
   - yarn smoke:silentcoverage
-  - yarn test-extension
+  - yarn test-clients
   - yarn test-viewer
   - yarn test-lantern
   - yarn i18n:checks

lighthouse-extension/bundle-builder.js → build/build-bundle.js

@@ -25,12 +25,12 @@ const COMMIT_HASH = require('child_process')
   .toString().trim();
 
 const audits = LighthouseRunner.getAuditList()
-    .map(f => '../lighthouse-core/audits/' + f.replace(/\.js$/, ''));
+    .map(f => './lighthouse-core/audits/' + f.replace(/\.js$/, ''));
 
 const gatherers = LighthouseRunner.getGathererList()
-    .map(f => '../lighthouse-core/gather/gatherers/' + f.replace(/\.js$/, ''));
+    .map(f => './lighthouse-core/gather/gatherers/' + f.replace(/\.js$/, ''));
 
-const locales = fs.readdirSync('../lighthouse-core/lib/i18n/locales/')
+const locales = fs.readdirSync(__dirname + '/../lighthouse-core/lib/i18n/locales/')
     .map(f => require.resolve(`../lighthouse-core/lib/i18n/locales/${f}`));
 
 /** @param {string} file */
@@ -81,7 +81,7 @@ async function browserifyFile(entryPath, distPath) {
   }
 
   // Expose the audits, gatherers, and computed artifacts so they can be dynamically loaded.
-  const corePath = '../lighthouse-core/';
+  const corePath = './lighthouse-core/';
   const driverPath = `${corePath}gather/`;
   audits.forEach(audit => {
     bundle = bundle.require(audit, {expose: audit.replace(corePath, '../')});
@@ -140,8 +140,23 @@ async function build(entryPath, distPath) {
   minifyScript(distPath);
 }
 
-module.exports = {
-  /** The commit hash for the current HEAD. */
-  COMMIT_HASH,
-  build,
-};
+/**
+ * @param {Array<string>} argv
+ */
+async function cli(argv) {
+  // Take paths relative to cwd and build.
+  const [entryPath, distPath] = argv.slice(2)
+    .map(filePath => path.resolve(process.cwd(), filePath));
+  build(entryPath, distPath);
+}
+
+// @ts-ignore Test if called from the CLI or as a module.
+if (require.main === module) {
+  cli(process.argv);
+} else {
+  module.exports = {
+    /** The commit hash for the current HEAD. */
+    COMMIT_HASH,
+    build,
+  };
+}

lighthouse-extension/build-extension.js → build/build-extension.js

@@ -8,51 +8,38 @@
 const fs = require('fs');
 
 const archiver = require('archiver');
-const del = require('del');
 const cpy = require('cpy');
 const makeDir = require('make-dir');
-const bundleBuilder = require('./bundle-builder.js');
-
-const distDir = 'dist';
-const manifestVersion = require(`./app/manifest.json`).version;
-
-// list of all consumers we build for (easier to understand which file is used for which)
-const CONSUMERS = {
-  DEVTOOLS: {
-    src: 'devtools-entry.js',
-    dist: 'lighthouse-dt-bundle.js',
-  },
-  EXTENSION: {
-    src: 'extension-entry.js',
-    dist: 'lighthouse-ext-bundle.js',
-  },
-  LIGHTRIDER: {
-    src: 'lightrider-entry.js',
-    dist: 'lighthouse-lr-bundle.js',
-  },
-};
+const bundleBuilder = require('./build-bundle.js');
+
+const sourceName = 'extension-entry.js';
+const distName = 'lighthouse-ext-bundle.js';
+
+const sourceDir = __dirname + '/../clients/extension';
+const distDir = __dirname + '/../dist/extension';
+
+const manifestVersion = require(`${sourceDir}/manifest.json`).version;
 
 /**
- * Browserify and minify scripts.
+ * Browserify and minify entry point.
  */
-function buildAll() {
-  return Object.values(CONSUMERS).map(consumer => {
-    const inFile = `app/src/${consumer.src}`;
-    const outFile = `dist/scripts/${consumer.dist}`;
-    return bundleBuilder.build(inFile, outFile);
-  });
+function buildEntryPoint() {
+  const inFile = `${sourceDir}/scripts/${sourceName}`;
+  const outFile = `${distDir}/scripts/${distName}`;
+  return bundleBuilder.build(inFile, outFile);
 }
 
 /**
  * Copy popup.js to dist folder, inlining the current commit hash along the way.
  * @return {Promise<void>}
  */
 async function copyPopup() {
-  let popupSrc = fs.readFileSync('app/src/popup.js', {encoding: 'utf8'});
+  let popupSrc = fs.readFileSync(`${sourceDir}/scripts/popup.js`, {encoding: 'utf8'});
   popupSrc = popupSrc.replace(/__COMMITHASH__/g, bundleBuilder.COMMIT_HASH);
 
-  await makeDir(`${distDir}/scripts`);
-  fs.writeFileSync(`${distDir}/scripts/popup.js`, popupSrc);
+  const popupDir = `${distDir}/scripts`;
+  await makeDir(popupDir);
+  fs.writeFileSync(`${popupDir}/popup.js`, popupSrc);
 }
 
 /**
@@ -65,8 +52,8 @@ async function copyAssets() {
     'images/**/*',
     'manifest.json',
     '_locales/**', // currently non-functional
-  ], `../${distDir}`, {
-    cwd: 'app',
+  ], distDir, {
+    cwd: sourceDir,
     parents: true,
   });
 }
@@ -77,17 +64,15 @@ async function copyAssets() {
  * @return {Promise<void>}
  */
 async function packageExtension() {
-  await del([
-    `${distDir}/scripts/${CONSUMERS.DEVTOOLS.dist}`,
-    `${distDir}/scripts/${CONSUMERS.LIGHTRIDER.dist}`,
-  ]);
+  const packagePath = `${distDir}/../extension-package`;
+  await makeDir(packagePath);
 
   return new Promise((resolve, reject) => {
     const archive = archiver('zip', {
       zlib: {level: 9},
     });
 
-    const outPath = `package/lighthouse-${manifestVersion}.zip`;
+    const outPath = `${packagePath}/lighthouse-${manifestVersion}.zip`;
     const writeStream = fs.createWriteStream(outPath);
     writeStream.on('finish', resolve);
     writeStream.on('error', reject);
@@ -105,7 +90,7 @@ async function run() {
   }
 
   await Promise.all([
-    ...buildAll(),
+    buildEntryPoint(),
     copyAssets(),
     copyPopup(),
   ]);

build/changelog-generator/index.js

@@ -10,6 +10,8 @@ const mainTemplate = readFileSync(resolve(__dirname, 'templates/template.hbs')).
 const headerPartial = readFileSync(resolve(__dirname, 'templates/header.hbs')).toString();
 const commitPartial = readFileSync(resolve(__dirname, 'templates/commit.hbs')).toString();
 
+/** @typedef {{type: string, header: string, hash?: string, message?: string, PR?: string}} Commit */
+
 const pullRequestRegex = /\(#(\d+)\)$/;
 const parserOpts = {
   headerPattern: /^(\w*)(?:\((.*)\))?: (.*)$/,
@@ -30,6 +32,7 @@ const writerOpts = {
   mainTemplate,
   headerPartial,
   commitPartial,
+  /** @param {Commit} commit */
   transform: commit => {
     if (typeof commit.hash === 'string') {
       commit.hash = commit.hash.substring(0, 7);
@@ -68,6 +71,7 @@ const writerOpts = {
     return commit;
   },
   groupBy: 'type',
+  /** @param {{title: string}} a @param {{title: string}} b */
   commitGroupsSort: (a, b) => {
     // put new audit on the top
     if (a.title === 'New Audits') {

lighthouse-extension/app/src/devtools-entry.js → clients/devtools-entry.js

@@ -5,11 +5,11 @@
  */
 'use strict';
 
-const lighthouse = require('../../../lighthouse-core/index');
-const RawProtocol = require('../../../lighthouse-core/gather/connections/raw');
+const lighthouse = require('../lighthouse-core/index.js');
+const RawProtocol = require('../lighthouse-core/gather/connections/raw.js');
 const log = require('lighthouse-logger');
 
-/** @typedef {import('../../../lighthouse-core/gather/connections/connection.js')} Connection */
+/** @typedef {import('../lighthouse-core/gather/connections/connection.js')} Connection */
 
 /**
  * Return a version of the default config, filtered to only run the specified

lighthouse-extension/app/src/lightrider-entry.js → clients/lightrider-entry.js

@@ -5,19 +5,19 @@
  */
 'use strict';
 
-const lighthouse = require('../../../lighthouse-core/index');
+const lighthouse = require('../lighthouse-core/index.js');
 
-const assetSaver = require('../../../lighthouse-core/lib/asset-saver.js');
-const LHError = require('../../../lighthouse-core/lib/lh-error.js');
-const preprocessor = require('../../../lighthouse-core/lib/proto-preprocessor.js');
+const assetSaver = require('../lighthouse-core/lib/asset-saver.js');
+const LHError = require('../lighthouse-core/lib/lh-error.js');
+const preprocessor = require('../lighthouse-core/lib/proto-preprocessor.js');
 
 /** @type {Record<'mobile'|'desktop', LH.Config.Json>} */
 const LR_PRESETS = {
-  mobile: require('../../../lighthouse-core/config/lr-mobile-config'),
-  desktop: require('../../../lighthouse-core/config/lr-desktop-config'),
+  mobile: require('../lighthouse-core/config/lr-mobile-config.js'),
+  desktop: require('../lighthouse-core/config/lr-desktop-config.js'),
 };
 
-/** @typedef {import('../../../lighthouse-core/gather/connections/connection.js')} Connection */
+/** @typedef {import('../lighthouse-core/gather/connections/connection.js')} Connection */
 
 /**
  * Run lighthouse for connection and provide similar results as in CLI.

lighthouse-extension/test/extension-test.js → clients/test/extension/extension-test.js

@@ -10,10 +10,11 @@
 const path = require('path');
 const assert = require('assert');
 const fs = require('fs');
-const puppeteer = require('../../node_modules/puppeteer/index.js');
+const puppeteer = require('../../../node_modules/puppeteer/index.js');
 
-const lighthouseExtensionPath = path.resolve(__dirname, '../dist');
-const config = require(path.resolve(__dirname, '../../lighthouse-core/config/default-config.js'));
+const lighthouseExtensionPath = path.resolve(__dirname, '../../../dist/extension');
+const config = require(path.resolve(__dirname,
+    '../../../lighthouse-core/config/default-config.js'));
 
 const getAuditsOfCategory = category => config.categories[category].auditRefs;
 

lighthouse-extension/test/popup-test.js → clients/test/extension/popup-test.js

@@ -9,9 +9,9 @@
 
 const path = require('path');
 const assert = require('assert');
-const puppeteer = require('../../node_modules/puppeteer/index.js');
+const puppeteer = require('../../../node_modules/puppeteer/index.js');
 
-const lighthouseExtensionPath = path.resolve(__dirname, '../dist');
+const lighthouseExtensionPath = path.resolve(__dirname, '../../../dist/extension');
 
 const defaultCategoriesStub = [
   {

lighthouse-extension/test/app/src/lightrider-entry-test.js → clients/test/lightrider-entry-test.js

@@ -6,8 +6,8 @@
 'use strict';
 
 const assert = require('assert');
-const lhBackground = require('../../../app/src/lightrider-entry.js');
-const LHError = require('../../../../lighthouse-core/lib/lh-error.js');
+const lhBackground = require('../lightrider-entry.js');
+const LHError = require('../../lighthouse-core/lib/lh-error.js');
 
 /* eslint-env mocha */
 

docs/releasing.md

@@ -38,7 +38,7 @@ yarn install-all
 
 # * Bump it *
 yarn version --no-git-tag-version
-# manually bump extension v in extension/app/manifest.json
+# manually bump extension v in clients/extension/manifest.json
 yarn update:sample-json
 
 # * Build it *
@@ -94,9 +94,15 @@ echo "Rebuild extension and viewer to get the latest, tagged master commit"
 yarn build-all;
 
 # zip the extension files
-cd lighthouse-extension; gulp package; cd ..
+node build/build-extension.js package; cd dist/extension-package/
 echo "Go here: https://chrome.google.com/webstore/developer/edit/blipmdconlkpinefehnmjammfjpmpbjk "
 echo "Upload the package zip to CWS dev dashboard"
+# Be in lighthouse-extension-owners group
+# Open <https://chrome.google.com/webstore/developer/dashboard>
+# Click _Edit_ on lighthouse
+# _Upload Updated Package_
+# Select `lighthouse-4.X.X.zip`
+# _Publish_ at the bottom
 
 echo "Verify the npm package won't include unncessary files"
 npm pack --dry-run

lighthouse-core/index.js

@@ -25,7 +25,7 @@ const LHError = require('./lib/lh-error.js');
  *
  *         lighthouse-cli \
  *                         -- core/index.js ----> runner.js ----> [Gather / Audit]
- *   lighthouse-extension /
+ *                clients /
  */
 
 /**

lighthouse-core/scripts/roll-to-devtools.sh

@@ -33,7 +33,7 @@ fi
 report_dir="lighthouse-core/report/html"
 fe_lh_dir="$frontend_dir/audits2/lighthouse"
 
-lh_bg_js="lighthouse-extension/dist/scripts/lighthouse-dt-bundle.js"
+lh_bg_js="dist/lighthouse-dt-bundle.js"
 lh_worker_dir="$frontend_dir/audits2_worker/lighthouse"
 
 # copy report files