Merge pull request #8150 from spalger/test/shardTestsToEaseMemoryPres…

…sure

add test sharding

package.json

@@ -195,6 +195,7 @@
     "makelogs": "3.0.2",
     "marked-text-renderer": "0.1.0",
     "mocha": "2.5.3",
+    "murmurhash3js": "3.0.1",
     "ncp": "2.0.0",
     "nock": "2.10.0",
     "npm": "3.10.3",

src/ui/public/test_harness/test_harness.js

@@ -10,8 +10,10 @@ import _ from 'lodash';
 import StackTraceMapper from 'ui/stack_trace_mapper';
 import { parse } from 'url';
 import $ from 'jquery';
+import { setupAutoRelease } from 'auto-release-sinon';
 import './test_harness.less';
 import 'ng_mock';
+import { setupTestSharding } from './test_sharding';
 
 /*** the vislib tests have certain style requirements, so lets make sure they are met ***/
 $('body').attr('id', 'test-harness-body'); // so we can make high priority selectors
@@ -24,13 +26,9 @@ Math.random = _.bindKey(new Nonsense(seed), 'frac');
 Math.random.nonsense = new Nonsense(seed);
 console.log('Random-ness seed: ' + seed);
 
-
-/*** Setup auto releasing stubs and spys ***/
-require('auto-release-sinon').setupAutoRelease(sinon, window.afterEach);
-
-
-/*** Make sure that angular-mocks gets setup in the global suite **/
-
+// Setup auto releasing stubs and spys
+setupAutoRelease(sinon, window.afterEach);
+setupTestSharding();
 
 /*** manually map error stack traces using the sourcemap ***/
 before(function () {

src/ui/public/test_harness/test_sharding/find_test_bundle_url.js

@@ -0,0 +1,20 @@
+/**
+ *  We don't have a lot of options for passing arguments to the page that karma
+ *  creates, so we tack some query string params onto the test bundle script url.
+ *
+ *  This function finds that url by looking for a script tag that has
+ *  the "/tests.bundle.js" segment
+ *
+ *  @return {string} url
+ */
+export function findTestBundleUrl() {
+  const scriptTags = document.querySelectorAll('script[src]');
+  const scriptUrls = [].map.call(scriptTags, el => el.getAttribute('src'));
+  const testBundleUrl = scriptUrls.find(url => url.includes('/tests.bundle.js'));
+
+  if (!testBundleUrl) {
+    throw new Error('test bundle url couldn\'t be found');
+  }
+
+  return testBundleUrl;
+}

src/ui/public/test_harness/test_sharding/get_shard_num.js

@@ -0,0 +1,28 @@
+import murmurHash3 from 'murmurhash3js';
+
+// murmur hashes are 32bit unsigned integers
+const MAX_HASH = Math.pow(2, 32);
+
+/**
+*  Determine the shard number for a suite by hashing
+*  its name and placing it based on the hash
+*
+*  @param {number} shardTotal - the total number of shards
+*  @param {string} suiteName - the suite name to hash
+*  @return {number} shardNum - 1-based shard number
+*/
+export function getShardNum(shardTotal, suiteName) {
+  const hashIntsPerShard = MAX_HASH / shardTotal;
+
+  const hashInt = murmurHash3.x86.hash32(suiteName);
+
+  // murmur3 produces 32bit integers, so we devide it by the number of chunks
+  // to determine which chunk the suite should fall in. +1 because the current
+  // chunk is 1-based
+  const shardNum = Math.floor(hashInt / hashIntsPerShard) + 1;
+
+  // It's not clear if hash32 can produce the MAX_HASH or not,
+  // but this just ensures that shard numbers don't go out of bounds
+  // and cause tests to be ignored unnecessarily
+  return Math.max(1, Math.min(shardNum, shardTotal));
+}

src/ui/public/test_harness/test_sharding/get_sharding_params_from_url.js

@@ -0,0 +1,26 @@
+import { parse as parseUrl } from 'url';
+
+/**
+ *  This function extracts the relevant "shards" and "shard_num"
+ *  params from the url.
+ *
+ *  @param {string} testBundleUrl
+ *  @return {object} params
+ *  @property {number} params.shards - the total number of shards
+ *  @property {number} params.shard_num - the current shard number, 1 based
+ */
+export function getShardingParamsFromUrl(url) {
+  const parsedUrl = parseUrl(url, true);
+  const parsedQuery = parsedUrl.query || {};
+
+  const params = {};
+  if (parsedQuery.shards) {
+    params.shards = parseInt(parsedQuery.shards, 10);
+  }
+
+  if (parsedQuery.shard_num) {
+    params.shard_num = parseInt(parsedQuery.shard_num, 10);
+  }
+
+  return params;
+}

src/ui/public/test_harness/test_sharding/index.js

@@ -0,0 +1 @@
+export { setupTestSharding } from './setup_test_sharding';

src/ui/public/test_harness/test_sharding/setup_test_sharding.js

@@ -0,0 +1,48 @@
+import { uniq, defaults } from 'lodash';
+
+import { findTestBundleUrl } from './find_test_bundle_url';
+import { getShardingParamsFromUrl } from './get_sharding_params_from_url';
+import { setupTopLevelDescribeFilter } from './setup_top_level_describe_filter';
+import { getShardNum } from './get_shard_num';
+
+const DEFAULT_PARAMS = {
+  shards: 1,
+  shard_num: 1
+};
+
+export function setupTestSharding() {
+  const pageUrl = window.location.href;
+  const bundleUrl = findTestBundleUrl();
+
+  // supports overriding params via the debug page
+  // url in dev mode
+  const params = defaults(
+    {},
+    getShardingParamsFromUrl(pageUrl),
+    getShardingParamsFromUrl(bundleUrl),
+    DEFAULT_PARAMS
+  );
+
+  const { shards: shardTotal, shard_num: shardNum } = params;
+  if (shardNum < 1 || shardNum > shardTotal) {
+    throw new TypeError(`shard_num param of ${shardNum} must be greater 0 and less than the total, ${shardTotal}`);
+  }
+
+  // track and log the number of ignored describe calls
+  const ignoredDescribeShards = [];
+  before(() => {
+    const ignoredCount = ignoredDescribeShards.length;
+    const ignoredFrom = uniq(ignoredDescribeShards).join(', ');
+    console.log(`Ignored ${ignoredCount} top-level suites from ${ignoredFrom}`);
+  });
+
+  // Filter top-level describe statements as they come
+  setupTopLevelDescribeFilter(describeName => {
+    const describeShardNum = getShardNum(shardTotal, describeName);
+    if (describeShardNum === shardNum) return true;
+    // track shard numbers that we ignore
+    ignoredDescribeShards.push(describeShardNum);
+  });
+
+  console.log(`ready to load tests for shard ${shardNum} of ${shardTotal}`);
+}

src/ui/public/test_harness/test_sharding/setup_top_level_describe_filter.js

@@ -0,0 +1,97 @@
+/**
+ *  Intercept all calls to mocha.describe() and determine
+ *  which calls make it through using a filter function.
+ *
+ *  The filter function is also only called for top-level
+ *  describe() calls; all describe calls nested within another
+ *  are allowed based on the filter value for the parent
+ *  describe
+ *
+ *  ## example
+ *
+ *  assume tests that look like this:
+ *
+ *  ```js
+ *  describe('section 1', () => {
+ *    describe('item 1', () => {
+ *
+ *    })
+ *  })
+ *  ```
+ *
+ *  If the filter function returned true for "section 1" then "item 1"
+ *  would automatically be defined. If it returned false for "section 1"
+ *  then "section 1" would be ignored and "item 1" would never be defined
+ *
+ *  @param {function} test - a function that takes the first argument
+ *                             passed to describe, the sections name, and
+ *                             returns true if the describe call should
+ *                             be delegated to mocha, any other value causes
+ *                             the describe call to be ignored
+ *  @return {undefined}
+ */
+export function setupTopLevelDescribeFilter(test) {
+  const originalDescribe = window.describe;
+
+  if (!originalDescribe) {
+    throw new TypeError('window.describe must be defined by mocha before test sharding can be setup');
+  }
+
+  /**
+   *  When describe is called it is likely to make additional, nested,
+   *  calls to describe. We track how deeply nested we are at any time
+   *  with a depth counter, `describeCallDepth`.
+   *
+   *  Before delegating a describe call to mocha we increment
+   *  that counter, and once mocha is done we decrement it.
+   *
+   *  This way, we can check if `describeCallDepth > 0` at any time
+   *  to know if we are already within a describe call.
+   *
+   *  ```js
+   *  // +1
+   *  describe('section 1', () => {
+   *    // describeCallDepth = 1
+   *    // +1
+   *    describe('item 1', () => {
+   *      // describeCallDepth = 2
+   *    })
+   *    // -1
+   *  })
+   *  // -1
+   *  // describeCallDepth = 0
+   *  ```
+   *
+   *  @type {Number}
+   */
+  let describeCallDepth = 0;
+  const ignoredCalls = [];
+
+  // ensure that window.describe isn't messed with by other code
+  Object.defineProperty(window, 'describe', {
+    configurable: false,
+    enumerable: true,
+    value: function describeInterceptor(describeName, describeBody) {
+      const context = this;
+
+      const isTopLevelCall = describeCallDepth === 0;
+      const shouldIgnore = isTopLevelCall && Boolean(test(describeName)) === false;
+      if (shouldIgnore) return;
+
+      /**
+       *  we wrap the delegation to mocha in a try/finally block
+       *  to ensure that our describeCallDepth counter stays up
+       *  to date even if the call throws an error.
+       *
+       *  note that try/finally won't actually catch the error, it
+       *  will continue to propogate up the call stack
+       */
+      try {
+        describeCallDepth += 1;
+        originalDescribe.call(context, describeName, describeBody);
+      } finally {
+        describeCallDepth -= 1;
+      }
+    }
+  });
+}

tasks/config/karma.js

@@ -1,5 +1,9 @@
+import { times, zipObject } from 'lodash';
+
+const TOTAL_CI_SHARDS = 4;
+
 module.exports = function (grunt) {
-  return {
+  const config = {
     options: {
       // base path that will be used to resolve all patterns (eg. files, exclude)
       basePath: '',
@@ -49,6 +53,72 @@ module.exports = function (grunt) {
           { type: 'text-summary' },
         ]
       }
-    }
+    },
   };
+
+  /**
+   *  ------------------------------------------------------------
+   *  CI sharding
+   *  ------------------------------------------------------------
+   *
+   *  Every test retains nearly all of the memory it causes to be allocated,
+   *  which has started to kill the test browser as the size of the test suite
+   *  increases. This is a deep-rooted problem that will take some serious
+   *  work to fix.
+   *
+   *  CI sharding is a short-term solution that splits the top-level describe
+   *  calls into different "shards" and instructs karma to only run one shard
+   *  at a time, reloading the browser in between each shard and forcing the
+   *  memory from the previous shard to be released.
+   *
+   *  ## how
+   *
+   *  Rather than modify the bundling process to produce multiple testing
+   *  bundles, top-level describe calls are sharded by their first argument,
+   *  the suite name.
+   *
+   *  The number of shards to create is controlled with the TOTAL_CI_SHARDS
+   *  constant defined at the top of this file.
+   *
+   *  ## controlling sharding
+   *
+   *  To control sharding in a specific karma configuration, the total number
+   *  of shards to create (?shards=X), and the current shard number
+   *  (&shard_num=Y), are added to the testing bundle url and read by the
+   *  test_harness/setup_test_sharding[1] module. This allows us to use a
+   *  different number of shards in different scenarios (ie. running
+   *  `npm run test:browser` runs the tests in a single shard, effectively
+   *  disabling sharding)
+   *
+   *  These same parameters can also be defined in the URL/query string of the
+   *  karma debug page (started when you run `npm run test:dev`).
+   *
+   *  ## debugging
+   *
+   *  It is *possible* that some tests will only pass if run after/with certain
+   *  other suites. To debug this, make sure that your tests pass in isolation
+   *  (by clicking the suite name on the karma debug page) and that it runs
+   *  correctly in it's given shard (using the `?shards=X&shard_num=Y` query
+   *  string params on the karma debug page). You can spot the shard number
+   *  a test is running in by searching for the "ready to load tests for shard X"
+   *  log message.
+   *
+   *  [1]: src/ui/public/test_harness/test_sharding/setup_test_sharding.js
+   */
+  times(TOTAL_CI_SHARDS, i => {
+    const n = i + 1;
+    config[`ciShard-${n}`] = {
+      singleRun: true,
+      options: {
+        files: [
+          'http://localhost:5610/bundles/commons.bundle.js',
+          `http://localhost:5610/bundles/tests.bundle.js?shards=${TOTAL_CI_SHARDS}&shard_num=${n}`,
+          'http://localhost:5610/bundles/commons.style.css',
+          'http://localhost:5610/bundles/tests.style.css'
+        ]
+      }
+    };
+  });
+
+  return config;
 };

tasks/jenkins.js

@@ -28,7 +28,7 @@ module.exports = function (grunt) {
     'eslint:source',
     'licenses',
     'test:server',
-    'test:browser',
+    'test:browser-ci',
     'test:api',
   ]);
 

tasks/test.js

@@ -1,4 +1,4 @@
-const _ = require('lodash');
+import _, { keys } from 'lodash';
 const visualRegression = require('../utilities/visual_regression');
 
 module.exports = function (grunt) {
@@ -22,7 +22,19 @@ module.exports = function (grunt) {
   );
 
   grunt.registerTask('test:server', [ 'esvm:test', 'simplemocha:all', 'esvm_shutdown:test' ]);
-  grunt.registerTask('test:browser', [ 'run:testServer', 'karma:unit' ]);
+  grunt.registerTask('test:browser', ['run:testServer', 'karma:unit']);
+  grunt.registerTask('test:browser-ci', () => {
+    const ciShardTasks = keys(grunt.config.get('karma'))
+      .filter(key => key.startsWith('ciShard-'))
+      .map(key => `karma:${key}`);
+
+    grunt.log.ok(`Running UI tests in ${ciShardTasks.length} shards`);
+
+    grunt.task.run([
+      'run:testServer',
+      ...ciShardTasks
+    ]);
+  });
   grunt.registerTask('test:coverage', [ 'run:testCoverageServer', 'karma:coverage' ]);
 
   grunt.registerTask('test:quick', [