Skip to content

Commit

Permalink
Merge pull request brave#10517 from brave/feature/perf-tests
Browse files Browse the repository at this point in the history
Automated perf tests
  • Loading branch information
bsclifton authored Oct 18, 2017
2 parents 106ba86 + 2c6eb55 commit 11f16cf
Show file tree
Hide file tree
Showing 12 changed files with 436 additions and 14 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
logs
*.log
npm-debug.log.*
cpu-profiles
*.cpuprofile

# Runtime data
pids
Expand Down
2 changes: 2 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ before_install:
- sh -e /etc/init.d/xvfb start
before_script:
- npm run download-sync-client
- curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash
script:
- npm run testsuite
branches:
Expand All @@ -30,6 +31,7 @@ env:
- CXX=g++-4.8 NODE_ENV=test TEST_DIR=misc-components
- CXX=g++-4.8 NODE_ENV=test TEST_DIR=navbar-components
- CXX=g++-4.8 NODE_ENV=test TEST_DIR=tab-components
- CXX=g++-4.8 NODE_ENV=test TEST_DIR=performance ARTIFACTS_REGION=us-east-1
addons:
apt:
sources:
Expand Down
10 changes: 10 additions & 0 deletions docs/performance.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
# Performance

## Automated tests

We have automated perf tests and you can find them in [tests/performance/](tests/performance/). These work just like the other spectron webdriver tests, however we turn on muon's `--debug` flag and then use the WebKit CPU profiler feature using [chrome-remote-interface](https://github.com/cyrus-and/chrome-remote-interface). This logs JavaScript CPU time and generates a .cpuprofile file.

You can run tests manually with `npm run test -- --grep='^Performance'`.

Travis runs perf tests on every run, and the generated .cpuprofiles are uploaded to S3 then consumed by https://github.com/brave/perfaderp (ask in our chat for access to our internal perfaderp instance) which tracks all test runtimes.

To debug problems, you can examine the .cpuprofiles directly with the browser's JavaScript Profiler tool. From the Brave Inspector, go to the top right Kabob menu -> More tools -> JavaScript Profiler then load the .cpuprofile. Now you can see an aggregate JS function tree with execution times.

## Reducer logging

One source of blocking lag is when the main process runs slow code in reducers. To look for this, enable reducer runtime logs by running Brave with the env variable `REDUCER_TIME_LOG_THRESHOLD={time in ms}`.
Expand Down
33 changes: 33 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@
"base64-js": "^1.2.0",
"chai": "^3.4.1",
"chai-as-promised": "^5.1.0",
"chrome-remote-interface": "^0.24.3",
"co-mocha": "^1.1.2",
"cross-env": "^3.1.4",
"css-loader": "~0.28.7",
Expand Down
29 changes: 16 additions & 13 deletions test/lib/brave.js
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ var exports = {
.waitForVisible(urlInput)
})

this.app.client.addCommand('waitForUrl', function (url) {
this.app.client.addCommand('waitForUrl', function (url, timeout = 5000, interval = 100) {
logVerbose('waitForUrl("' + url + '")')
return this.waitUntil(function () {
return this.tabByUrl(url).then((response) => {
Expand All @@ -355,7 +355,7 @@ var exports = {
logVerbose('tabByUrl("' + url + '") => false')
return false
})
}, 5000, null, 100)
}, timeout, null, interval)
})

this.app.client.addCommand('waitForSelectedText', function (text) {
Expand Down Expand Up @@ -703,7 +703,7 @@ var exports = {
}).then((response) => response.value)
})

this.app.client.addCommand('newTab', function (createProperties = {}) {
this.app.client.addCommand('newTab', function (createProperties = {}, activateIfOpen = false, isRestore = false) {
return this
.execute(function (createProperties) {
return devTools('appActions').createTabRequested(createProperties)
Expand Down Expand Up @@ -733,20 +733,18 @@ var exports = {
return this.execute(function (siteDetail) {
return devTools('appActions').addBookmark(siteDetail)
}, siteDetail).then((response) => response.value)
.waitForBookmarkEntry(waitUrl, false)
.waitForBookmarkEntry(waitUrl)
})

this.app.client.addCommand('waitForBookmarkEntry', function (location, waitForTitle = true) {
logVerbose('waitForBookmarkEntry("' + location + '", "' + waitForTitle + '")')
this.app.client.addCommand('waitForBookmarkEntry', function (location) {
logVerbose('waitForBookmarkEntry("' + location + '")')
return this.waitUntil(function () {
return this.getAppState().then((val) => {
const ret = val.value && val.value.bookmarks && Array.from(Object.values(val.value.bookmarks)).find(
(bookmark) => bookmark.location === location &&
(!waitForTitle || (waitForTitle && bookmark.title)))
logVerbose('waitForBookmarkEntry("' + location + ', ' + waitForTitle + '") => ' + ret)
const ret = val.value.cache.bookmarkLocation.hasOwnProperty(location)
logVerbose('waitForBookmarkEntry("' + location + '") => ' + ret)
return ret
})
}, 5000, null, 100)
}, 10000, null, 100)
})

/**
Expand Down Expand Up @@ -1149,7 +1147,10 @@ var exports = {
})
},

startApp: function () {
/**
* @param {Array=} extraArgs
*/
startApp: function (extraArgs) {
if (process.env.KEEP_BRAVE_USER_DATA_DIR) {
console.log('BRAVE_USER_DATA_DIR=' + userDataDir)
}
Expand All @@ -1158,6 +1159,8 @@ var exports = {
BRAVE_USER_DATA_DIR: userDataDir,
SPECTRON: true
}
let args = ['./', '--enable-logging', '--v=1']
if (extraArgs) { args = args.concat(extraArgs) }
this.app = new Application({
quitTimeout: 0,
waitTimeout: exports.defaultTimeout,
Expand All @@ -1167,7 +1170,7 @@ var exports = {
? 'node_modules/electron-prebuilt/dist/brave.exe'
: './node_modules/.bin/electron',
env,
args: ['./', '--enable-logging', '--v=1'],
args,
requireName: 'devTools'
})
return this.app.start()
Expand Down
108 changes: 108 additions & 0 deletions test/lib/profilerUtil.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

'use strict'

const Brave = require('./brave')
const CDP = require('chrome-remote-interface')
const fs = require('fs')

const LOG_FOLDER = './cpu-profiles'

// Private
// ===

/**
* Derive main process remote debug port based on webdriver args.
* @param {Spectron.Application} app e.g. Brave.app
* @returns {number}
*/
const getDebugPort = function (app) {
if (!app.args) { return }
for (let arg of app.args) {
if (arg.indexOf('--debug') === -1 && arg.indexOf('--inspect') === -1) {
continue
}
const regex = /([0-9]+)$/
const result = arg.match(regex)
if (!result[1]) { continue }
return parseInt(result[1])
}
}

// Public
// ===

/**
* Connect to a remote instance using Chrome Debugging Protocol.
* See https://github.com/cyrus-and/chrome-remote-interface#cdpoptions-callback
*/
const initCDP = function * () {
const port = getDebugPort(Brave.app)
if (!port) {
throw new Error("Could not determine RDP port from webdriver app args. Did you start with Brave.startApp(['--debug={inspectPort}'] ?")
}
const cdp = yield CDP({port})
Brave.cdp = cdp
}

const startProfiler = function * () {
yield initCDP()
yield Brave.cdp.Profiler.enable()
yield Brave.cdp.Profiler.setSamplingInterval({interval: 100})
yield Brave.cdp.Profiler.start()
}

/**
* @param logTag {string=} Optional file prefix
* @returns filename to which CPU profile was written
*/
const stopProfiler = function * (logTag = '') {
const cdpProfilerResult = yield Brave.cdp.Profiler.stop()
if (!fs.existsSync(LOG_FOLDER)) {
console.log(`Creating directory ${LOG_FOLDER}`)
fs.mkdirSync(LOG_FOLDER)
}
const fileContent = JSON.stringify(cdpProfilerResult.profile, null, 2)
const filename = `${logTag}-${new Date().toISOString()}.cpuprofile`
const path = `${LOG_FOLDER}/${filename}`
fs.writeFile(path, fileContent)
console.log(`Wrote CPU profile data to: ${path}`)
return filename
}

/**
* Profile a function.
* @param {function} fn
*/
const profile = function * (fn, logTag) {
yield startProfiler()
yield fn()
yield stopProfiler(logTag)
}

const uploadTravisArtifacts = function * () {
if (!process.env.TRAVIS) { return }
console.log('Uploading Travis artifacts...')
const execute = require('../../tools/lib/execute')
const command = `artifacts upload ${LOG_FOLDER} --target-paths "$TRAVIS_REPO_SLUG/$TRAVIS_BRANCH/$TRAVIS_BUILD_NUMBER--$TRAVIS_COMMIT"`
yield new Promise((resolve, reject) => {
execute(command, process.env, (err) => {
if (err) {
console.error('Failed to upload artifacts', err)
process.exit(1)
return reject(err)
}
resolve()
})
})
}

module.exports = {
initCDP,
startProfiler,
stopProfiler,
profile,
uploadTravisArtifacts
}
53 changes: 53 additions & 0 deletions test/lib/userProfiles.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
const Immutable = require('immutable')
const niceware = require('niceware')

const addBookmarksN = function (total) {
if (!total || total > 65536) {
throw new Error('Can only add up to 65536 bookmarks.')
}
return function * (client) {
const data = []
const buffer = new Buffer(2)
for (let n = 0; n < total; n++) {
buffer.writeUInt16BE(n)
const string = niceware.bytesToPassphrase(buffer)[0]
data.push({
location: `https://www.${string}.com`,
title: string,
parentFolderId: 0
})
}
const lastBookmark = data.pop()
const immutableData = Immutable.fromJS(data)
yield client.waitForBrowserWindow()
.addBookmarks(immutableData)
.addBookmark(lastBookmark)
}
}
const addBookmarks4000 = addBookmarksN(4000)

const addTabsN = function (total) {
return function * (client) {
const data = []
const buffer = new Buffer(2)
for (let n = 0; n < total; n++) {
buffer.writeUInt16BE(n)
const string = niceware.bytesToPassphrase(buffer)[0]
data.push({
active: false,
discarded: true,
url: `https://www.${string}.com`
})
}
yield client.waitForBrowserWindow()
for (let datum of data) {
yield client.newTab(datum, false, true) // isRestore
}
}
}
const addTabs50 = addTabsN(50)

module.exports = {
addBookmarks4000,
addTabs50
}
Loading

0 comments on commit 11f16cf

Please sign in to comment.