Add Tachometer Benchmarking

.gitignore

@@ -4,12 +4,12 @@ styles/**/*.css.ts
 
 node_modules
 npm-debug.log
-.nvmrc
 .DS_STORE
 .chrome
 .firefox
-.vscode
 .npmrc
+.nvmrc
+.tsbuildinfo
 
 documentation/api-docs/component_data.json
 documentation/api-docs/*.html
@@ -18,4 +18,8 @@ documentation/dist
 package-deps.json
 spectrum-web-components.build.log
 spectrum-web-components.build.error.log
-TESTS.xml
+TESTS.xml
+
+/test/benchmark/**/*.js
+/test/benchmark/**/*.js.map
+/test/benchmark/**/*.d.ts 

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+ "files.exclude": {
+ "test/benchmark/**/*.js": true,
+ "test/benchmark/**/*.d.ts": true,
+ "test/benchmark/**/*.js.map": true,
+ "test/**/*.spec.js": true,
+ "test/**/*.spec.d.ts": true,
+ "test/**/*.spec.js.map": true
+ }
+}

README.md

@@ -54,6 +54,17 @@ npm run test
 
 During development you may wish to use `npm run test:watch` to automatically build and re-run the test suites.
 
+## Benchmarking
+
+You can acquire current runtimes for the individual elements with:
+
+```
+npm run build:tests
+npm run test:bench
+```
+
+This will run the defined [Tachometer](https://www.npmjs.com/package/tachometer) tests and report the current runtime cost of each individual element. When not making changes to the benchmarks thy have been built on your local machine, you can stip `npm run build:tests` for later passes.
+
 ## Development within Adobe
 
 Due to the internal `@adobe` npm registry served by Artifactory, when developing this module internally it is necessary to provide an `.npmrc` local to this project which forces the use of the public `@adobe` scope on public npm. Therefore if you are an Adobe employee working on this project add the following to a `.npmrc` file in this folder:

package.json

@@ -28,6 +28,7 @@
  "process-icons": "node ./scripts/process-icons.js",
  "prebuild": "rimraf lib",
  "build": "gulp",
+ "build:tests": "tsc --build test/tsconfig.json && tsc --build test/tsconfig-node.json",
  "watch": "gulp watch",
  "serve": "node ./scripts/serve.js",
  "serve:watch": "npm run build && (npm run serve & npm run watch)",
@@ -36,6 +37,7 @@
  "docs:start": "gulp docsWatchCompile",
  "lint": "bash ./scripts/lint.sh",
  "test": "karma start --single-run",
+ "test:bench": "node test/benchmark/cli.js",
  "test:watch": "karma start"
  },
  "husky": {
@@ -64,6 +66,8 @@
  "@storybook/core": "5.0.11",
  "@storybook/polymer": "5.0.11",
  "@types/chai": "^4.1.7",
+ "@types/command-line-args": "^5.0.0",
+ "@types/command-line-usage": "^5.0.1",
  "@types/common-tags": "^1.4.0",
  "@types/karma-chai": "^0.1.1",
  "@types/mocha": "^5.2.6",
@@ -115,8 +119,8 @@
  "karma-sourcemap-loader": "^0.3.7",
  "karma-webpack": "^4.0.2",
  "linebyline": "^1.3.0",
- "lit-element": "^2.1.0",
- "lit-html": "^1.0.0",
+ "lit-element": "^2.2.1",
+ "lit-html": "^1.1.1",
  "lodash": "^4.17.15",
  "markdown-loader": "^5.1.0",
  "marked": "^0.7.0",
@@ -146,6 +150,7 @@
  "stylelint": "^9.10.1",
  "stylelint-config-prettier": "^4.0.0",
  "stylelint-config-standard": "^18.2.0",
+ "tachometer": "^0.4.9",
  "ts-loader": "^6.0.0",
  "typedoc": "^0.15.0",
  "typescript": "^3.4.1",

stories/slider.stories.ts

@@ -38,7 +38,7 @@ storiesOf('Slider', module)
  max="${max}"
  label="${label}"
  id="opacity-slider"
- />
+ ></sp-slider>
  </div>
  `;
  })
@@ -53,7 +53,7 @@ storiesOf('Slider', module)
  min="0"
  max="20"
  label="${label}"
- />
+ ></sp-slider>
  </div>
  `;
  })
@@ -68,7 +68,7 @@ storiesOf('Slider', module)
  max="20"
  label="Color"
  id="color-slider"
- />
+ ></sp-slider>
  </div>
  `;
  })
@@ -84,7 +84,7 @@ storiesOf('Slider', module)
  max="20"
  label="Color"
  id="color-slider"
- />
+ ></sp-slider>
  </div>
  `;
  })
@@ -101,7 +101,7 @@ storiesOf('Slider', module)
  max="20"
  label="Color"
  id="color-slider"
- />
+ ></sp-slider>
  </div>
  `;
  })
@@ -119,7 +119,7 @@ storiesOf('Slider', module)
  max="${max}"
  label="Opacity"
  id="opacity-slider-opacity"
- />
+ ></sp-slider>
  </div>
  <div style="width: 500px; margin: 20px;">
  <sp-slider
@@ -129,7 +129,7 @@ storiesOf('Slider', module)
  max="${max}"
  label="Lightness"
  id="opacity-slider-lightness"
- />
+ ></sp-slider>
  </div>
  <div style="width: 500px; margin: 20px;">
  <sp-slider
@@ -139,7 +139,7 @@ storiesOf('Slider', module)
  max="${max}"
  label="Saturation"
  id="opacity-slider-saturation"
- />
+ ></sp-slider>
  </div>
  `;
  });

test/benchmark/banner/test-basic.ts

@@ -0,0 +1,10 @@
+import '../../../lib/banner';
+import { html } from 'lit-html';
+import { measureFixtureCreation } from '../helpers';
+
+measureFixtureCreation(html`
+ <sp-banner type="info">
+ <div slot="header">Info Text</div>
+ <div slot="content">Info Content</div>
+ </sp-banner>
+`);

test/benchmark/bench-runner.html

@@ -0,0 +1,17 @@
+<html>
+ <head>
+ <link
+ rel="shortcut icon"
+ href="data:image/x-icon;,"
+ type="image/x-icon"
+ />
+ </head>
+ <body>
+ <script type="module">
+ const params = new URLSearchParams(window.location.search);
+ const pack = params.get('package');
+ const bench = params.get('bench');
+ import(`./${pack}/${bench}.js`);
+ </script>
+ </body>
+</html>

test/benchmark/card/test-basic.ts

@@ -0,0 +1,11 @@
+import '../../../lib/card';
+import { html } from 'lit-html';
+import { measureFixtureCreation } from '../helpers';
+
+measureFixtureCreation(html`
+ <sp-card variant="gallery" title="Card Title" subtitle="JPG">
+ <img slot="preview" src="https://picsum.photos/532/192" />
+ <div slot="description">10/15/18</div>
+ <div slot="footer">Footer</div>
+ </sp-card>
+`);

test/benchmark/checkbox/test-basic.ts

@@ -0,0 +1,9 @@
+import '../../../lib/checkbox';
+import { html } from 'lit-html';
+import { measureFixtureCreation } from '../helpers';
+
+measureFixtureCreation(html`
+ <sp-checkbox id="checkbox0" tabindex="5">
+ Component
+ </sp-checkbox>
+`);

test/benchmark/cli.ts

@@ -0,0 +1,148 @@
+import { main } from 'tachometer/lib/cli';
+import { readdirSync } from 'fs';
+import { join as pathjoin } from 'path';
+import * as commandLineArgs from 'command-line-args';
+import * as commandLineUsage from 'command-line-usage';
+
+const optionDefinitions: commandLineUsage.OptionDefinition[] = [
+ {
+ name: 'help',
+ description: 'Show documentation',
+ type: Boolean,
+ defaultValue: false,
+ },
+ {
+ name: 'package',
+ description:
+ 'Select which individual packages to benchmark.\ne.g.' +
+ ' "-p button radio icon-button".\n(default runs all)',
+ alias: 'p',
+ type: String,
+ multiple: true,
+ defaultValue: [],
+ },
+ {
+ name: 'remote',
+ description:
+ 'Remote location to point tachometer.\ne.g. if running' +
+ ' a remote SSH Selenium tunnel on port 4444:\n' +
+ '-r http://localhost:4444/wd/hub.\n(default runs locally)',
+ alias: 'r',
+ type: String,
+ defaultValue: '',
+ },
+ {
+ name: 'sample-size',
+ description:
+ 'Minimum number of times to run each benchmark.\n(default 50)',
+ alias: 'n',
+ type: Number,
+ defaultValue: 50,
+ },
+ {
+ name: 'browser',
+ description:
+ 'Which browsers to launch in automatic mode.' +
+ '\n(default chrome)',
+ alias: 'b',
+ type: String,
+ defaultValue: 'chrome',
+ },
+];
+
+interface Options {
+ help: boolean;
+ package: string[];
+ remote: string;
+ 'sample-size': string;
+ browser: string;
+}
+
+(async () => {
+ const opts = commandLineArgs(optionDefinitions) as Options;
+
+ if (opts.help) {
+ console.log(
+ commandLineUsage([
+ {
+ header: 'benchmark runner',
+ content: `Runs benchmarks for SWC`,
+ },
+ {
+ header: 'Usage',
+ content: `Run all benchmarks for all SWC components:
+$ node test/benchmark/cli
+Run all benchmarks for specific components:
+$ node test/benchmark/cli -p button textfield
+$ node test/benchmark/cli -p button -p textfield
+Run benchmarks on remote Selenium server or SSH tunnel:
+$ node test/benchmark/cli -r http://localhost:4444/wd/hub
+Run benchmarks n times on each package:
+$ node test/benchmark/cli -n 20
+`,
+ },
+ {
+ header: 'Options',
+ optionList: optionDefinitions,
+ },
+ ])
+ );
+ return;
+ }
+
+ let packages: string[] = [];
+
+ if (opts.package.length) {
+ packages = opts.package;
+ } else {
+ packages = readdirSync(pathjoin('test', 'benchmark'), {
+ withFileTypes: true,
+ })
+ .filter((dirEntry) => dirEntry.isDirectory())
+ .map((dirEntry) => dirEntry.name);
+ }
+
+ const printResults: string[] = [];
+ for (const packageName of packages) {
+ const runCommands: string[] = [];
+
+ const benchmarks = readdirSync(
+ pathjoin('test', 'benchmark', packageName),
+ { withFileTypes: true }
+ )
+ .filter(
+ (dirEntry) => dirEntry.isFile() && dirEntry.name.endsWith('.js')
+ )
+ .map((dirEntry) => dirEntry.name.replace(/\.js$/, ''));
+
+ for (const benchmark of benchmarks) {
+ runCommands.push(
+ `${packageName}:${benchmark}=test/benchmark/bench-runner.html` +
+ `?bench=${benchmark}` +
+ `&package=${packageName}`
+ );
+ }
+
+ const statResults = await main([
+ ...runCommands,
+ '--measure=global',
+ `--browser=${opts.browser}${opts.remote ? `@${opts.remote}` : ''}`,
+ `--sample-size=${opts['sample-size']}`,
+ ]);
+
+ if (!statResults) {
+ return;
+ }
+
+ for (const statResult of statResults) {
+ const name = statResult.result.name;
+ const low = statResult.stats.meanCI.low.toFixed(2);
+ const high = statResult.stats.meanCI.high.toFixed(2);
+ printResults.push(`${name}: ${low}ms - ${high}ms`);
+ }
+ }
+
+ for (const printResult of printResults) {
+ console.log(printResult);
+ }
+})();