[functionalTestRunner] replace intern (elastic#10910)

* [functional_test_runner] replace functional testing tools with custom/pluggable solution

* [functional_test_runner] Convert unit tests to commonjs format

* [functional_test_runner] Fix dashboard test in wrong mode

* [functional_test_runner] Add dashboardLandingPage test subject

* [functional_test_runner] Get Visualize page object

* [functional_test_runner] Fix outdated references

* [functional_test_runner] Fix more outdated refs

* [functional_test_runner] Remove duplicate tests

* [functional_test_runner] Improve test readability

* [functional_test_runner] 😞 So many duplicate methods

* [functional_test_runner] Move mgmt `before` outside toplevel describe

* [functional_test_runner] Settings page obj missing methods

* [functional_test_runner] Add improvements from @gammon

* [functional_test_runner] Fix return statements in async funcs

* [functional_test_runner] Move before() to correct scope

* [functional_test_runner] Add after() hooks to remove index patterns

* [functional_test_runner] Attempt to fix vertical bar chart tests

* [functional_test_runner] Clean up

* [functional_test_runner] Reinstate unit tests

* [functional_test_runner] Set default loglevel back to info

* [functional_test_runner] Replace `context`s with `describe`s

* [functional_test_runner] Better error handling

* [functional_test_runner] Add in new Tile Map tests

* Incorporate changes from master

* [functional_test_runner] validate that every test file has a single top-level suite

* Update contributing doc with link to full doc

* [docs] Spelling and grammar fixes

* docs: writing and running functional tests

* [docs] Move plugin doc to plugin area

* [docs] Housekeeping. Doc in wrong place

* [docs] Remove dup doc file

* [grunt] Only run mocha_setup when running tests, not every grunt task

CONTRIBUTING.md

@@ -308,12 +308,7 @@ npm run test:ui:runner
 
 ##### Browser Automation Notes
 
-- Using Page Objects pattern (https://theintern.github.io/intern/#writing-functional-test)
-- At least the initial tests for the Settings, Discover, and Visualize tabs all depend on a very specific set of logstash-type data (generated with makelogs).  Since that is a static set of data, all the Discover and Visualize tests use a specific Absolute time range.  This guarantees the same results each run.
-- These tests have been developed and tested with Chrome and Firefox browser.  In theory, they should work on all browsers (that's the benefit of Intern using Leadfoot).
-- These tests should also work with an external testing service like https://saucelabs.com/ or https://www.browserstack.com/ but that has not been tested.
-- https://theintern.github.io/
-- https://theintern.github.io/leadfoot/module-leadfoot_Element.html
+[Read about the `FunctionalTestRunner`](https://www.elastic.co/guide/en/kibana/current/development-functional-tests.html) to learn more about how you can run and develop functional tests for Kibana core and plugins.
 
 ### Building OS packages
 
@@ -374,4 +369,4 @@ Remember, someone is blocked by a pull awaiting review, make it count. Be thorou
 1. **Hand it off** If you're the first reviewer and everything looks good but the changes are more than a few lines, hand the pull to someone else to take a second look. Again, try to find the right person to assign it to.
 1. **Merge the code** When everything looks good, put in a `LGTM` (looks good to me) comment. Merge into the target branch. Check the labels on the pull to see if backporting is required, and perform the backport if so.
 
-Thank you so much for reading our guidelines! :tada:
+Thank you so much for reading our guidelines! :tada:

Gruntfile.js

@@ -57,7 +57,7 @@ module.exports = function (grunt) {
     init: true,
     config: config,
     loadGruntTasks: {
-      pattern: ['grunt-*', '@*/grunt-*', 'gruntify-*', '@*/gruntify-*', 'intern']
+      pattern: ['grunt-*', '@*/grunt-*', 'gruntify-*', '@*/gruntify-*']
     }
   });
 

docs/development/core-development.asciidoc

@@ -5,11 +5,14 @@
 * <<development-dependencies>>
 * <<development-modules>>
 * <<development-elasticsearch>>
+* <<development-functional-tests>>
 
 include::core/development-basepath.asciidoc[]
 
 include::core/development-dependencies.asciidoc[]
 
 include::core/development-modules.asciidoc[]
 
-include::plugin/development-elasticsearch.asciidoc[]
+include::core/development-elasticsearch.asciidoc[]
+
+include::core/development-functional-tests.asciidoc[]

docs/development/plugin-development.asciidoc

@@ -8,8 +8,11 @@ The Kibana plugin interfaces are in a state of constant development.  We cannot
 
 * <<development-plugin-resources>>
 * <<development-uiexports>>
+* <<development-plugin-functional-tests>>
 
 
 include::plugin/development-plugin-resources.asciidoc[]
 
 include::plugin/development-uiexports.asciidoc[]
+
+include::plugin/development-plugin-functional-tests.asciidoc[]

docs/development/plugin/development-plugin-functional-tests.asciidoc

@@ -0,0 +1,91 @@
+[[development-plugin-functional-tests]]
+=== Functional Tests for Plugins
+
+Plugins use the `FunctionalTestRunner` by running it out of the Kibana repo. Ensure that your Kibana Development Environment is setup properly before continuing.
+
+[float]
+==== Writing your own configuration
+
+Every project or plugin should have its own `FunctionalTestRunner` config file. Just like Kibana's, this config file will define all of the test files to load, providers for Services and PageObjects, as well as configuration options for certain services.
+
+To get started copy and paste this example to `test/functional/config.js`:
+
+["source","js"]
+-----------
+import { resolve } from 'path';
+import { MyServiceProvider } from './services/my_service';
+import { MyAppPageProvider } from './services/my_app_page;
+
+// allow overriding the default kibana directory
+// using the KIBANA_DIR environment variable
+const KIBANA_CONFIG_PATH = resolve(process.env.KIBANA_DIR || '../kibana', 'test/functional/config.js');
+
+// the default export of config files must be a config provider
+// that returns an object with the projects config values
+export default async function ({ readConfigFile }) {
+
+  // read the Kibana config file so that we can utilize some of
+  // its services and PageObjects
+  const kibanaConfig = await readConfigFile(KIBANA_CONFIG_PATH);
+
+  return {
+    // list paths to the files that contain your plugins tests
+    testFiles: [
+      resolve(__dirname, './my_test_file.js'),
+    ],
+
+    // define the name and providers for services that should be
+    // available to your tests. If you don't specify anything here
+    // only the built-in services will be avaliable
+    services: {
+      ...kibanaConfig.get('services'),
+      myService: MyServiceProvider,
+    },
+
+    // just like services, PageObjects are defined as a map of
+    // names to Providers. Merge in Kibana's or pick specific ones
+    pageObjects: {
+      management: kibanaConfig.get('pageObjects.management'),
+      myApp: MyAppPageProvider,
+    },
+
+    // the apps section defines the urls that
+    // `PageObjects.common.navigateTo(appKey)` will use.
+    // Merge urls for your plugin with the urls defined in
+    // Kibana's config in order to use this helper
+    apps: {
+      ...kibanaConfig.get('apps'),
+      myApp: {
+        pathname: '/app/my_app',
+      }
+    },
+
+    // choose where esArchiver should load archives from
+    esArchiver: {
+      directory: resolve(__dirname, './es_archives'),
+    },
+
+    // choose where screenshots should be saved
+    screenshots: {
+      directory: resolve(__dirname, './tmp/screenshots'),
+    }
+
+    // more settings, like timeouts, mochaOpts, etc are
+    // defined in the config schema. See {blob}src/functional_test_runner/lib/config/schema.js[src/functional_test_runner/lib/config/schema.js]
+  };
+}
+
+-----------
+
+From the root of your repo you should now be able to run the `FunctionalTestRunner` script from your plugin project.
+
+["source","shell"]
+-----------
+node ../kibana/scripts/functional_test_runner
+-----------
+
+[float]
+==== Using esArchiver
+
+We're working on documentation for this, but for now the best place to look is the original {pull}10359[pull request].
+

docs/index.asciidoc

@@ -16,6 +16,7 @@ release-state can be: released | prerelease | unreleased
 :issue:          {repo}issues/
 :pull:           {repo}pull/
 :commit:         {repo}commit/
+:blob:           {repo}blob/{branch}/
 :security:       https://www.elastic.co/community/security/
 
 

package.json

@@ -126,8 +126,8 @@
     "expose-loader": "0.7.0",
     "extract-text-webpack-plugin": "0.8.2",
     "file-loader": "0.8.4",
-    "font-awesome": "4.4.0",
     "flot-charts": "0.8.3",
+    "font-awesome": "4.4.0",
     "glob": "5.0.13",
     "glob-all": "3.0.1",
     "good-squeeze": "2.1.0",
@@ -209,8 +209,9 @@
     "chance": "1.0.6",
     "cheerio": "0.22.0",
     "chokidar": "1.6.0",
-    "chromedriver": "2.24.1",
+    "chromedriver": "2.28.0",
     "classnames": "2.2.5",
+    "digdug": "1.6.3",
     "enzyme": "2.7.0",
     "enzyme-to-json": "1.4.5",
     "eslint": "3.11.1",
@@ -239,7 +240,6 @@
     "html-loader": "0.4.3",
     "husky": "0.8.1",
     "image-diff": "1.6.0",
-    "intern": "3.2.3",
     "istanbul-instrumenter-loader": "0.1.3",
     "jest": "19.0.0",
     "jest-cli": "19.0.0",
@@ -252,6 +252,7 @@
     "karma-mocha": "0.2.0",
     "karma-safari-launcher": "0.1.1",
     "keymirror": "0.1.1",
+    "leadfoot": "1.7.1",
     "license-checker": "5.1.2",
     "load-grunt-config": "0.19.2",
     "makelogs": "3.2.3",

scripts/functional_test_runner.js

@@ -0,0 +1,2 @@
+require('../src/optimize/babel/register');
+require('../src/functional_test_runner/cli');

src/core_plugins/kibana/public/dashboard/listing/dashboard_listing.html

@@ -12,7 +12,10 @@
   </div>
 </kbn-top-nav>
 
-<div class="kuiViewContent kuiViewContent--constrainedWidth">
+<div
+  class="kuiViewContent kuiViewContent--constrainedWidth"
+  data-test-subj="dashboardLandingPage"
+>
   <!-- ControlledTable -->
   <div class="kuiViewContentItem kuiControlledTable kuiVerticalRhythm">
     <!-- ToolBar -->

src/es_archiver/actions/load.js

@@ -9,7 +9,7 @@ import {
   isGzip,
   createStats,
   prioritizeMappings,
-  getArchiveFiles,
+  readDirectory,
   createParseArchiveStreams,
   createCreateIndexStream,
   createIndexDocRecordsStream,
@@ -19,7 +19,7 @@ export async function loadAction({ name, skipExisting, client, dataDir, log }) {
   const inputDir = resolve(dataDir, name);
   const stats = createStats(name, log);
 
-  const files = prioritizeMappings(await getArchiveFiles(inputDir));
+  const files = prioritizeMappings(await readDirectory(inputDir));
   for (const filename of files) {
     log.info('[%s] Loading %j', name, filename);
 

src/es_archiver/actions/rebuild_all.js

@@ -1,7 +1,6 @@
 import { resolve } from 'path';
 import {
   rename,
-  readdir,
   createReadStream,
   createWriteStream
 } from 'fs';
@@ -14,18 +13,18 @@ import {
 
 import {
   prioritizeMappings,
-  getArchiveFiles,
+  readDirectory,
   isGzip,
   createParseArchiveStreams,
   createFormatArchiveStreams,
 } from '../lib';
 
 export async function rebuildAllAction({ dataDir, log }) {
-  const archiveNames = await fromNode(cb => readdir(dataDir, cb));
+  const archiveNames = await readDirectory(dataDir);
 
   for (const name of archiveNames) {
     const inputDir = resolve(dataDir, name);
-    const files = prioritizeMappings(await getArchiveFiles(inputDir));
+    const files = prioritizeMappings(await readDirectory(inputDir));
     for (const filename of files) {
       log.info('[%s] Rebuilding %j', name, filename);
 

src/es_archiver/actions/unload.js

@@ -9,7 +9,7 @@ import {
   isGzip,
   createStats,
   prioritizeMappings,
-  getArchiveFiles,
+  readDirectory,
   createParseArchiveStreams,
   createFilterRecordsStream,
   createDeleteIndexStream
@@ -19,7 +19,7 @@ export async function unloadAction({ name, client, dataDir, log }) {
   const inputDir = resolve(dataDir, name);
   const stats = createStats(name, log);
 
-  const files = prioritizeMappings(await getArchiveFiles(inputDir));
+  const files = prioritizeMappings(await readDirectory(inputDir));
   for (const filename of files) {
     log.info('[%s] Unloading indices from %j', name, filename);
 

src/es_archiver/cli.js

@@ -6,12 +6,14 @@
 
 import { resolve } from 'path';
 import { readFileSync } from 'fs';
+import { format as formatUrl } from 'url';
 
 import { Command } from 'commander';
 import elasticsearch from 'elasticsearch';
 
 import { EsArchiver } from './es_archiver';
-import { createLog } from './lib';
+import { createToolingLog } from '../utils';
+import { readConfigFile } from '../functional_test_runner';
 
 const cmd = new Command('node scripts/es_archiver');
 
@@ -20,6 +22,7 @@ cmd
   .option('--es-url [url]', 'url for elasticsearch')
   .option(`--dir [path]`, 'where archives are stored')
   .option('--verbose', 'turn on verbose logging')
+  .option('--config [path]', 'path to a functional test config file to use for default values')
   .on('--help', () => {
     console.log(readFileSync(resolve(__dirname, './cli_help.txt'), 'utf8'));
   });
@@ -49,9 +52,16 @@ if (missingCommand) {
 
 async function execute(operation, ...args) {
   try {
-    const log = createLog(cmd.verbose ? 'debug' : 'info');
+    const log = createToolingLog(cmd.verbose ? 'debug' : 'info');
     log.pipe(process.stdout);
 
+    if (cmd.config) {
+      // load default values from the specified config file
+      const config = await readConfigFile(log, resolve(cmd.config));
+      if (!cmd.esUrl) cmd.esUrl = formatUrl(config.get('servers.elasticsearch'));
+      if (!cmd.dir) cmd.dir = config.get('esArchiver.directory');
+    }
+
     // log and count all validation errors
     let errorCount = 0;
     const error = (msg) => {
@@ -61,10 +71,10 @@ async function execute(operation, ...args) {
 
     if (!operation) error('Missing or invalid command');
     if (!cmd.esUrl) {
-      error('You must specify either --es-url flag');
+      error('You must specify either --es-url or --config flags');
     }
     if (!cmd.dir) {
-      error('You must specify either --dir flag');
+      error('You must specify either --dir or --config flags');
     }
 
     // if there was a validation error display the help
@@ -84,7 +94,7 @@ async function execute(operation, ...args) {
       const esArchiver = new EsArchiver({
         log,
         client,
-        dataDir: resolve(cmd.dir)
+        dataDir: resolve(cmd.dir),
       });
       await esArchiver[operation](...args);
     } finally {