[ftr] make room for more projects (elastic#11848)

Squashed commit of the following:

commit 659ea986fdeb9a5ff2ca1fa5360cccb01c671ede
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 09:19:22 2017 -0700

    [test/*/fixtures] rename es_archives to es_archiver

commit d3667457c78e88e2d6974f3c38dd0fe61b846b91
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 08:22:03 2017 -0700

    [ftr/config] fix default directory value functions

commit 9a6a2cc0b295e2281e83da04fcea40e9d6f00781
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 07:30:52 2017 -0700

    fix import paths

commit fcb65a877d54b5b1d36b8c81f1264b36845e826e
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 21:39:57 2017 -0700

    [grunt/ftr] use named exports for configs

commit 7d7f38c7615cdbf8eb0119efc0f2a5188bca8792
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:50:04 2017 -0700

    [test] remove unnecessary directory definitions

commit 0c28984669768482f0a2ee7fc2800d5bcaf49025
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:46:29 2017 -0700

    [ftr/config] make default directories relative to config path

commit cd2f33612624cacffec138797f3fc0f4ecb46cca
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:32:12 2017 -0700

    [test/common] put server config into common

commit 7851ed811a236576c63bd20850b3ef2099be2a4e
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:18:20 2017 -0700

    [grunt] "deprecate" test:api:runner task

commit b2ac4c26593a1947c94f0168191fe8123ff74122
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:15:41 2017 -0700

    [ftr] accept the project name as an unnamed arg

commit 47e292894fc70c0a04883403c50c5d2ae0738d76
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:56:34 2017 -0700

    [ftr/grunt] convert ftr task to multi-task with config

commit 83375855f88e5e7b3fa8b6a1c5d24a9f54766ce5
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:54:54 2017 -0700

    [test/functional] move fixtures into test/functional project

commit 05994e9c92cf134c58f831c285b3b522a801acbc
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:02:51 2017 -0700

    [src/test_utils] merge with test/utils directory

commit c77ee5ed36b8b7eadf876cb6d9482a49dfc92b66
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:53:00 2017 -0700

    [test/api_integration] migrate api tests to functional test runner

commit ca328c34648dd7e07f70e1844e07cfc392e41103
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:50:12 2017 -0700

    [esArchiver] refresh modified indices after load

commit cde74a540850fd97578f441d6dccaefd1444e656
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:46:48 2017 -0700

    [test/functional] move shared services into test/common

commit 0ea2646aea5817f6d1595e6ae0d356c426f138f0
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 22:51:23 2017 -0700

    [scripts/mocha] run _mocha script when debugging

commit 1cc80600d90e318d4738920aa557d124075a4570
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 22:48:12 2017 -0700

    [ftr/config] allow child config files to have no testFiles config

commit 2bb6c957443b18cebc419baa6f9db301c8f4dc4f
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 21:52:26 2017 -0700

    [ftr] move screenshots into test/functional

.gitignore

@@ -12,10 +12,10 @@ target
 .idea
 *.iml
 *.log
-/test/screenshots/diff
-/test/screenshots/failure
-/test/screenshots/session
-/test/screenshots/visual_regression_gallery.html
+/test/*/screenshots/diff
+/test/*/screenshots/failure
+/test/*/screenshots/session
+/test/*/screenshots/visual_regression_gallery.html
 /html_docs
 /esvm
 .htpasswd

Gruntfile.js

@@ -12,7 +12,6 @@ module.exports = function (grunt) {
     plugins: __dirname + '/src/core_plugins',
     server: __dirname + '/src/server',
     target: __dirname + '/target', // location of the compressed build targets
-    testUtilsDir: __dirname + '/src/test_utils',
     configFile: __dirname + '/src/config/kibana.yml',
 
     karmaBrowser: (function () {

package.json

@@ -137,7 +137,7 @@
     "inert": "4.0.2",
     "jade": "1.11.0",
     "jade-loader": "0.7.1",
-    "joi": "6.6.1",
+    "joi": "10.4.1",
     "jquery": "2.2.4",
     "js-yaml": "3.4.1",
     "json-loader": "0.5.3",
@@ -272,7 +272,7 @@
     "source-map": "0.5.6",
     "source-map-support": "0.2.10",
     "strip-ansi": "^3.0.1",
-    "supertest": "1.2.0",
+    "supertest": "3.0.0",
     "supertest-as-promised": "2.0.2",
     "tree-kill": "1.1.0",
     "webpack-dev-server": "1.14.1"

scripts/mocha.js

@@ -1 +1,2 @@
-require('mocha/bin/mocha');
+require('../src/optimize/babel/register');
+require('../test/scripts/run_mocha');

src/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js

@@ -4,7 +4,7 @@ import sinon from 'sinon';
 import expect from 'expect.js';
 import url from 'url';
 
-import serverConfig from '../../../../../test/server_config';
+import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 import { ensureEsVersion } from '../ensure_es_version';
 
 describe('plugins/elasticsearch', () => {
@@ -22,7 +22,7 @@ describe('plugins/elasticsearch', () => {
             status: {
               red: sinon.stub()
             },
-            url: url.format(serverConfig.servers.elasticsearch)
+            url: url.format(esTestServerUrlParts)
           }
         }
       };

src/core_plugins/elasticsearch/lib/__tests__/health_check.js

@@ -8,10 +8,10 @@ const NoConnections = require('elasticsearch').errors.NoConnections;
 import mappings from './fixtures/mappings';
 import healthCheck from '../health_check';
 import kibanaVersion from '../kibana_version';
-import serverConfig from '../../../../../test/server_config';
+import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 
-const esPort = serverConfig.servers.elasticsearch.port;
-const esUrl = url.format(serverConfig.servers.elasticsearch);
+const esPort = esTestServerUrlParts.port;
+const esUrl = url.format(esTestServerUrlParts);
 
 describe('plugins/elasticsearch', () => {
   describe('lib/health_check', function () {

src/core_plugins/elasticsearch/lib/__tests__/routes.js

@@ -1,6 +1,6 @@
 import { format } from 'util';
 
-import * as kbnTestServer from '../../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../../test_utils/kbn_server';
 import { fromRoot } from '../../../../utils';
 
 describe('plugins/elasticsearch', function () {

src/core_plugins/kibana/server/lib/__tests__/manage_uuid.js

@@ -1,6 +1,6 @@
 import expect from 'expect.js';
 import sinon from 'sinon';
-import * as kbnTestServer from '../../../../../../test/utils/kbn_server.js';
+import * as kbnTestServer from '../../../../../test_utils/kbn_server.js';
 import { fromRoot } from '../../../../../utils';
 import manageUuid from '../manage_uuid';
 

src/core_plugins/tests_bundle/index.js

@@ -71,7 +71,7 @@ export default (kibana) => {
         ng_mock$: fromRoot('src/core_plugins/dev_mode/public/ng_mock'),
         'angular-mocks$': require.resolve('./webpackShims/angular-mocks'),
         fixtures: fromRoot('src/fixtures'),
-        test_utils: fromRoot('src/test_utils'),
+        test_utils: fromRoot('src/test_utils/public'),
       }
     }
   });

src/es_archiver/actions/load.js

@@ -31,8 +31,14 @@ export async function loadAction({ name, skipExisting, client, dataDir, log }) {
     ]);
   }
 
+  const indicesToRefresh = [];
   stats.forEachIndex((index, { docs }) => {
     log.info('[%s] Indexed %d docs into %j', name, docs.indexed, index);
+    indicesToRefresh.push(index);
+  });
+
+  await client.indices.refresh({
+    index: indicesToRefresh
   });
 
   return stats.toJSON();

src/functional_test_runner/__tests__/lib/kibana.js

@@ -1,6 +1,6 @@
 import { resolve } from 'path';
 
-import { createServer } from '../../../../test/utils/kbn_server';
+import { createServer } from '../../../test_utils/kbn_server';
 
 export async function startupKibana({ port, esUrl }) {
   const server = createServer({

src/functional_test_runner/cli.js

@@ -6,7 +6,6 @@ import { createToolingLog } from '../utils';
 import { createFunctionalTestRunner } from './functional_test_runner';
 
 const cmd = new Command('node scripts/functional_test_runner');
-
 const resolveConfigPath = v => resolve(process.cwd(), v);
 const defaultConfigPath = resolveConfigPath('test/functional/config.js');
 
@@ -20,13 +19,6 @@ cmd
   .option('--debug', 'Run in debug mode', false)
   .parse(process.argv);
 
-if (!cmd.config) {
-  console.log('');
-  console.log('  --config is a required parameter');
-  console.log('');
-  process.exit(1);
-}
-
 let logLevel = 'info';
 if (cmd.silent) logLevel = 'silent';
 if (cmd.quiet) logLevel = 'error';

src/functional_test_runner/lib/config/__tests__/fixtures/config.1.js

@@ -0,0 +1,7 @@
+export default function () {
+  return {
+    testFiles: [
+      'config.1'
+    ]
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.2.js

@@ -0,0 +1,10 @@
+export default async function ({ readConfigFile }) {
+  const config1 = await readConfigFile(require.resolve('./config.1.js'));
+
+  return {
+    testFiles: [
+      ...config1.get('testFiles'),
+      'config.2'
+    ]
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.3.js

@@ -0,0 +1,11 @@
+export default async function ({ readConfigFile }) {
+  const config4 = await readConfigFile(require.resolve('./config.4'));
+  return {
+    testFiles: [
+      'baz'
+    ],
+    screenshots: {
+      ...config4.get('screenshots')
+    }
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.4.js

@@ -0,0 +1,7 @@
+export default function () {
+  return {
+    screenshots: {
+      directory: 'bar'
+    }
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.invalid.js

@@ -0,0 +1,5 @@
+export default async function () {
+  return {
+    foo: 'bar'
+  };
+}

src/functional_test_runner/lib/config/__tests__/read_config_file.js

@@ -0,0 +1,58 @@
+import expect from 'expect.js';
+
+import { createToolingLog } from '../../../../utils';
+import { readConfigFile } from '../read_config_file';
+import { Config } from '../config';
+
+const log = createToolingLog().resume();
+
+describe('readConfigFile()', () => {
+  it('reads config from a file, returns an instance of Config class', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.1'));
+    expect(config).to.be.a(Config);
+    expect(config.get('testFiles')).to.eql([
+      'config.1'
+    ]);
+  });
+
+  it('merges setting overrides into log', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.1'), {
+      screenshots: {
+        directory: 'foo.bar'
+      }
+    });
+
+    expect(config.get('screenshots.directory')).to.be('foo.bar');
+  });
+
+  it('supports loading config files from within config files', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.2'));
+    expect(config.get('testFiles')).to.eql([
+      'config.1',
+      'config.2',
+    ]);
+  });
+
+  it('throws if settings are invalid', async () => {
+    try {
+      await readConfigFile(log, require.resolve('./fixtures/config.invalid'));
+      throw new Error('expected readConfigFile() to fail');
+    } catch (err) {
+      expect(err.message).to.match(/"foo"/);
+    }
+  });
+
+  it('throws if config does not define testFiles', async () => {
+    try {
+      await readConfigFile(log, require.resolve('./fixtures/config.4'));
+      throw new Error('expected readConfigFile() to fail');
+    } catch (err) {
+      expect(err.message).to.match(/"testFiles"/);
+    }
+  });
+
+  it('does not throw if child config file does not have any testFiles', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.3'));
+    expect(config.get('screenshots.directory')).to.be('bar');
+  });
+});

src/functional_test_runner/lib/config/config.js

@@ -6,8 +6,25 @@ import { schema } from './schema';
 const $values = Symbol('values');
 
 export class Config {
-  constructor(settings) {
-    const { error, value } = schema.validate(settings);
+  constructor(options = {}) {
+    const {
+      settings = {},
+      primary = false,
+      path,
+    } = options;
+
+    if (!path) {
+      throw new TypeError('path is a required option');
+    }
+
+    const { error, value } = schema.validate(settings, {
+      abortEarly: false,
+      context: {
+        primary: !!primary,
+        path,
+      }
+    });
+
     if (error) throw error;
     this[$values] = value;
   }

src/functional_test_runner/lib/config/read_config_file.js

@@ -3,29 +3,37 @@ import { defaultsDeep } from 'lodash';
 import { Config } from './config';
 import { transformDeprecations } from './transform_deprecations';
 
-export async function readConfigFile(log, configFile, settingOverrides = {}) {
-  log.debug('Loading config file from %j', configFile);
+async function getSettingsFromFile(log, path, settingOverrides) {
+  log.debug('Loading config file from %j', path);
 
-  const configModule = require(configFile);
+  const configModule = require(path);
   const configProvider = configModule.__esModule
     ? configModule.default
     : configModule;
 
-  const settings = defaultsDeep(
+  const settingsWithDefaults = defaultsDeep(
     {},
     settingOverrides,
     await configProvider({
       log,
-
-      // give a version of the readConfigFile function to
-      // the config file that already has has the logger bound
-      readConfigFile: async (...args) => (
-        await readConfigFile(log, ...args)
-      )
+      async readConfigFile(...args) {
+        return new Config({
+          settings: await getSettingsFromFile(log, ...args),
+          primary: false,
+          path,
+        });
+      }
     })
   );
 
-  return new Config(transformDeprecations(settings, msg => {
-    log.error(msg);
-  }));
+  const logDeprecation = (...args) => log.error(...args);
+  return transformDeprecations(settingsWithDefaults, logDeprecation);
+}
+
+export async function readConfigFile(log, path, settingOverrides) {
+  return new Config({
+    settings: await getSettingsFromFile(log, path, settingOverrides),
+    primary: true,
+    path,
+  });
 }

src/functional_test_runner/lib/config/schema.js

@@ -1,3 +1,5 @@
+import { resolve, dirname } from 'path';
+
 import Joi from 'joi';
 
 import { ConsoleReporterProvider } from '../reporters';
@@ -18,8 +20,20 @@ const urlPartsSchema = () => Joi.object().keys({
   hash: Joi.string().regex(/^\//, 'start with a /')
 }).default();
 
+const defaultRelativeToConfigPath = path => {
+  const makeDefault = (locals, options) => (
+    resolve(dirname(options.context.path), path)
+  );
+  makeDefault.description = `<config.js directory>/${path}`;
+  return makeDefault;
+};
+
 export const schema = Joi.object().keys({
-  testFiles: Joi.array().items(Joi.string()).required(),
+  testFiles: Joi.array().items(Joi.string()).when('$primary', {
+    is: true,
+    then: Joi.required(),
+    otherwise: Joi.default([]),
+  }),
 
   services: Joi.object().pattern(
     ID_PATTERN,
@@ -74,11 +88,11 @@ export const schema = Joi.object().keys({
 
   // settings for the esArchiver module
   esArchiver: Joi.object().keys({
-    directory: Joi.string().required()
-  }),
+    directory: Joi.string().default(defaultRelativeToConfigPath('fixtures/es_archiver'))
+  }).default(),
 
   // settings for the screenshots module
   screenshots: Joi.object().keys({
-    directory: Joi.string().required()
-  }),
+    directory: Joi.string().default(defaultRelativeToConfigPath('screenshots'))
+  }).default(),
 }).default();

src/server/__tests__/base_path.js

@@ -1,6 +1,6 @@
 import expect from 'expect.js';
 
-import * as kbnTestServer from '../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../test_utils/kbn_server';
 const basePath = '/kibana';
 
 describe('Server basePath config', function () {

src/server/http/__tests__/index.js

@@ -1,5 +1,5 @@
 import expect from 'expect.js';
-import * as kbnTestServer from '../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../test_utils/kbn_server';
 import { fromRoot } from '../../../utils';
 
 describe('routes', function () {

src/server/http/__tests__/version_check.js

@@ -1,7 +1,7 @@
 import expect from 'expect.js';
 import { fromNode } from 'bluebird';
 import { resolve } from 'path';
-import * as kbnTestServer from '../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../test_utils/kbn_server';
 
 const src = resolve.bind(null, __dirname, '../../../../src');