Merge pull request elastic#7529 from cjcenizal/7523/visual-regression…

…-visualize

Create visual_regression test suite.

package.json

@@ -48,6 +48,7 @@
     "test:ui:runner": "grunt test:ui:runner",
     "test:server": "grunt test:server",
     "test:coverage": "grunt test:coverage",
+    "test:visualRegression": "grunt test:visualRegression",
     "build": "grunt build",
     "release": "grunt release",
     "start": "sh ./bin/kibana --dev",

tasks/config/intern.js

@@ -13,7 +13,13 @@ module.exports = function (grunt) {
     api: {
       options: {
         runType: 'client',
-        config: 'test/api_intern'
+        config: 'test/intern_api'
+      }
+    },
+    visualRegression: {
+      options: {
+        runType: 'runner',
+        config: 'test/intern_visual_regression'
       }
     }
   };

tasks/test.js

@@ -2,8 +2,18 @@ const _ = require('lodash');
 const visualRegression = require('../utilities/visual_regression');
 
 module.exports = function (grunt) {
+  grunt.registerTask('test:visualRegression', [
+    'intern:visualRegression:takeScreenshots',
+    'test:visualRegression:buildGallery'
+  ]);
+
+  grunt.registerTask('test:visualRegression:takeScreenshots', [
+    'clean:screenshots',
+    'intern:visualRegression'
+  ]);
+
   grunt.registerTask(
-    'test:visualRegression',
+    'test:visualRegression:buildGallery',
     'Compare screenshots and generate diff images.',
     function () {
       const done = this.async();
@@ -72,8 +82,7 @@ module.exports = function (grunt) {
     grunt.task.run(_.compact([
       !grunt.option('quick') && 'eslint:source',
       'licenses',
-      'test:quick',
-      'test:visualRegression'
+      'test:quick'
     ]));
   });
 

test/functional/index.js

@@ -3,18 +3,18 @@ define(function (require) {
 
   const bdd = require('intern!bdd');
   const intern = require('intern');
-  const earliestBeforeCbs = [];
+  const initCallbacks = [];
 
-  function onEarliestBefore(cb) {
-    earliestBeforeCbs.push(cb);
+  function onInit(callback) {
+    initCallbacks.push(callback);
   }
 
-  global.__kibana__intern__ = { intern, bdd, onEarliestBefore };
+  global.__kibana__intern__ = { intern, bdd, onInit };
 
   bdd.describe('kibana', function () {
     bdd.before(function () {
-      earliestBeforeCbs.forEach(cb => {
-        cb.call(this);
+      initCallbacks.forEach(callback => {
+        callback.call(this);
       });
     });
 

test/intern.js

@@ -1,8 +1,6 @@
 define(function (require) {
-  var serverConfig = require('intern/dojo/node!./server_config');
-  var _ = require('intern/dojo/node!lodash');
-
-  return _.assign({
+  const serverConfig = require('intern/dojo/node!./server_config');
+  return Object.assign({
     debug: true,
     capabilities: {
       'selenium-version': '2.53.0',

test/intern_visual_regression.js

@@ -0,0 +1,24 @@
+define(function (require) {
+  const serverConfig = require('intern/dojo/node!./server_config');
+  return Object.assign({
+    debug: true,
+    capabilities: {
+      'selenium-version': '2.53.0',
+      // must match URL in tasks/config/downloadSelenium.js
+      'idle-timeout': 99
+    },
+    environments: [{
+      browserName: 'chrome'
+    }],
+    tunnelOptions: serverConfig.servers.webdriver,
+    functionalSuites: [
+      'test/visual_regression/index'
+    ],
+
+    excludeInstrumentation: /.*/,
+
+    defaultTimeout: 90000,
+    defaultTryTimeout: 40000, // tryForTime could include multiple 'find timeouts'
+    defaultFindTimeout: 10000  // this is how long we try to find elements on page
+  }, serverConfig);
+});

test/support/index.js

@@ -23,36 +23,64 @@ exports.scenarioManager = new ScenarioManager(url.format(exports.config.servers.
 exports.esClient = new EsClient(url.format(exports.config.servers.elasticsearch));
 exports.bdd = new BddWrapper(kbnInternVars.bdd);
 
-defineDelayedExport('remote', (suite) => suite.remote);
-defineDelayedExport('common', () => new Common());
-defineDelayedExport('discoverPage', () => new DiscoverPage());
-defineDelayedExport('headerPage', () => new HeaderPage());
-defineDelayedExport('settingsPage', () => new SettingsPage());
-defineDelayedExport('visualizePage', () => new VisualizePage());
-defineDelayedExport('dashboardPage', () => new DashboardPage());
-defineDelayedExport('shieldPage', () => new ShieldPage());
-defineDelayedExport('consolePage', () => new ConsolePage());
-defineDelayedExport('elasticDump', () => new ElasticDump());
+const delayedExports = [{
+  name: 'remote',
+  exportProvider: (suite) => suite.remote
+}, {
+  name: 'common',
+  exportProvider: () => new Common()
+}, {
+  name: 'discoverPage',
+  exportProvider: () => new DiscoverPage()
+}, {
+  name: 'headerPage',
+  exportProvider: () => new HeaderPage()
+}, {
+  name: 'settingsPage',
+  exportProvider: () => new SettingsPage()
+}, {
+  name: 'visualizePage',
+  exportProvider: () => new VisualizePage()
+}, {
+  name: 'dashboardPage',
+  exportProvider: () => new DashboardPage()
+}, {
+  name: 'shieldPage',
+  exportProvider: () => new ShieldPage()
+}, {
+  name: 'consolePage',
+  exportProvider: () => new ConsolePage()
+}, {
+  name: 'elasticDump',
+  exportProvider: () => new ElasticDump()
+}];
 
-// creates an export for values that aren't actually avaialable until
-// until tests start to run. These getters will throw errors if the export
+delayedExports.forEach(exportConfig => {
+  delayExportUntilInit(
+    exportConfig.name,
+    exportConfig.exportProvider
+  );
+});
+
+// Creates an export for values that aren't actually avaialable until
+// tests start to run. These getters will throw errors if the export
 // is accessed before it's available, hopefully making debugging easier.
-// Once the first before() handler is called the onEarliestBefore() handlers
+// Once the first before() handler is called the onInit() call
 // will fire and rewrite all of these exports to be their correct value.
-function defineDelayedExport(name, define) {
+function delayExportUntilInit(name, provideExport) {
   Object.defineProperty(exports, name, {
     configurable: true,
     get() {
       throw new TypeError(
-        'remote is not available until tests start to run. Move your ' +
+        'Remote is not available until tests start to run. Move your ' +
         'usage of the import inside a test setup hook or a test itself.'
       );
     }
   });
 
-  kbnInternVars.onEarliestBefore(function () {
+  kbnInternVars.onInit(function defineExport() {
     Object.defineProperty(exports, name, {
-      value: define(this),
+      value: provideExport(this),
     });
   });
 }