Support for multiple Elasticsearch nodes and sniffing (#21928)

* Node sniffing

* update monitoring

* cleaning

* remove tests

* rewrite tests using the proxy

* fix monitoring mocha tests

* fix payload

* return full error

* default interval false

* add sniff settings to monitoring

* add docs for sniff settings

* re-add index search integration test

* catch parse error

* capital

* fix merge

* hapi upgrade

* return

* pass by prefix

* rm empty test fille

* split error handling

* fix merge

* update recent elasticsearch.url references

* prettier vs eslint

* transport.request, include query params

* mega comment

* revert elasticsearchUrl

* safe hosts

* more docs es.url references

* remove unused ml/esServerUrl

* log if both set

config/kibana.yml

@@ -24,8 +24,8 @@
 # The Kibana server's name.  This is used for display purposes.
 #server.name: "your-hostname"
 
-# The URL of the Elasticsearch instance to use for all your queries.
-#elasticsearch.url: "http://localhost:9200"
+# The URLs of the Elasticsearch instances to use for all your queries.
+#elasticsearch.hosts: ["http://localhost:9200"]
 
 # When this setting's value is true Kibana uses the hostname specified in the server.host
 # setting. When the value of this setting is false, Kibana uses the hostname of the host

docs/monitoring/monitoring-kibana.asciidoc

@@ -67,7 +67,7 @@ and {ref}/cluster-update-settings.html[Cluster update settings].
 more information, see <<monitoring-settings-kb,Monitoring settings in {kib}>>.
 
 . Identify where to send monitoring data. {kib} automatically
-sends metrics to the {es} cluster specified in the `elasticsearch.url` setting
+sends metrics to the {es} cluster specified in the `elasticsearch.hosts` setting
 in the `kibana.yml` file. This property has a default value of
 `http://localhost:9200`. +
 +

docs/monitoring/monitoring-troubleshooting.asciidoc

@@ -34,8 +34,8 @@ The *Monitoring* page in {kib} is empty.
 
 . Confirm that {kib} is seeking monitoring data from the appropriate {es} URL.
 By default, data is retrieved from the cluster specified in the 
-`elasticsearch.url` setting in the `kibana.yml` file. If you want to retrieve it
-from a different monitoring cluster, set `xpack.monitoring.elasticsearch.url`.
+`elasticsearch.hosts` setting in the `kibana.yml` file. If you want to retrieve it
+from a different monitoring cluster, set `xpack.monitoring.elasticsearch.hosts`.
 See <<monitoring-settings-kb>>.
 
 . Confirm that there is monitoring data available at that URL. It is stored in

docs/monitoring/viewing-metrics.asciidoc

@@ -25,8 +25,8 @@ a gold license, you can send data from multiple clusters to the same monitoring
 cluster and view them all through the same instance of {kib}. 
 
 By default, data is retrieved from the cluster specified in the 
-`elasticsearch.url` value in the `kibana.yml` file. If you want to retrieve it 
-from a different cluster, set `xpack.monitoring.elasticsearch.url`. 
+`elasticsearch.hosts` value in the `kibana.yml` file. If you want to retrieve it
+from a different cluster, set `xpack.monitoring.elasticsearch.hosts`.
 
 To learn more about typical monitoring architectures, 
 see {stack-ov}/how-monitoring-works.html[How monitoring works] and 

docs/security/securing-communications/index.asciidoc

@@ -63,13 +63,13 @@ must have a proxy that provides an HTTPS endpoint for {es}.
 
 --
 
-.. Specify the HTTPS protocol in the `elasticsearch.url` setting in the {kib}
+.. Specify the HTTPS protocol in the `elasticsearch.hosts` setting in the {kib}
 configuration file, `kibana.yml`:
 +
 --
 [source,yaml]
 --------------------------------------------------------------------------------
-elasticsearch.url: "https://<your_elasticsearch_host>.com:9200"
+elasticsearch.hosts: ["https://<your_elasticsearch_host>.com:9200"]
 --------------------------------------------------------------------------------
 --
 
@@ -100,13 +100,13 @@ must have a proxy that provides an HTTPS endpoint for {es}.
 
 --
 
-.. Specify the HTTPS URL in the `xpack.monitoring.elasticsearch.url` setting in 
+.. Specify the HTTPS URL in the `xpack.monitoring.elasticsearch.hosts` setting in 
 the {kib} configuration file, `kibana.yml`
 +
 --
 [source,yaml]
 --------------------------------------------------------------------------------
-xpack.monitoring.elasticsearch.url: "https://<your_monitoring_cluster>:9200"
+xpack.monitoring.elasticsearch.hosts: ["https://<your_monitoring_cluster>:9200"]
 --------------------------------------------------------------------------------
 --
 

docs/settings/monitoring-settings.asciidoc

@@ -37,9 +37,9 @@ Set to `true` (default) to enable {monitoring} in {kib}. Unlike the
 monitoring back-end does not run and {kib} stats are not sent to the monitoring
 cluster.
 
-`xpack.monitoring.elasticsearch.url`::
+`xpack.monitoring.elasticsearch.hosts`::
 Specifies the location of the {es} cluster where your monitoring data is stored.
-By default, this is the same as the `elasticsearch.url`. This setting enables
+By default, this is the same as `elasticsearch.hosts`. This setting enables
 you to use a single {kib} instance to search and visualize data in your
 production cluster as well as monitor data sent to a dedicated monitoring
 cluster.

docs/setup/docker.asciidoc

@@ -102,7 +102,7 @@ services:
     image: {docker-image}
     environment:
       SERVER_NAME: kibana.example.org
-      ELASTICSEARCH_URL: http://elasticsearch.example.org
+      ELASTICSEARCH_HOSTS: http://elasticsearch.example.org
 ----------------------------------------------------------
 
 Since environment variables are translated to CLI arguments, they take
@@ -117,7 +117,7 @@ images:
 [horizontal]
 `server.name`:: `kibana`
 `server.host`:: `"0"`
-`elasticsearch.url`:: `http://elasticsearch:9200`
+`elasticsearch.hosts`:: `http://elasticsearch:9200`
 `xpack.monitoring.ui.container.elasticsearch.enabled`:: `true`
 
 NOTE: The setting `xpack.monitoring.ui.container.elasticsearch.enabled` is not

docs/setup/production.asciidoc

@@ -82,10 +82,10 @@ http.port: 9200
 transport.host: <external ip>
 transport.tcp.port: 9300 - 9400
 --------
-. Make sure Kibana is configured to point to your local client node. In `kibana.yml`, the `elasticsearch.url` should be set to
-`localhost:9200`.
+. Make sure Kibana is configured to point to your local client node. In `kibana.yml`, the `elasticsearch.hosts` setting should be set to
+`["localhost:9200"]`.
 +
 --------
 # The Elasticsearch instance to use for all your queries.
-elasticsearch.url: "http://localhost:9200"
+elasticsearch.hosts: ["http://localhost:9200"]
 --------

docs/setup/settings.asciidoc

@@ -39,6 +39,12 @@ Elasticsearch. This value must be a positive integer.
 
 `elasticsearch.shardTimeout:`:: *Default: 30000* Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
 
+`elasticsearch.sniffInterval:`:: *Default: false* Time in milliseconds between requests to check Elasticsearch for an updated list of nodes.
+
+`elasticsearch.sniffOnStart:`:: *Default: false* Attempt to find other Elasticsearch nodes on startup.
+
+`elasticsearch.sniffOnConectionFault:`:: *Default: false* Update the list of Elasticsearch nodes immediately following a connection fault.
+
 `elasticsearch.ssl.certificate:` and `elasticsearch.ssl.key:`:: Optional settings that provide the paths to the PEM-format SSL
 certificate and key files. These files are used to verify the identity of Kibana to Elasticsearch and are required when `xpack.ssl.verification_mode` in Elasticsearch is set to either `certificate` or `full`.
 
@@ -57,8 +63,8 @@ requests for end-users being executed as the identity tied to the configured cer
 `elasticsearch.startupTimeout:`:: *Default: 5000* Time in milliseconds to wait for Elasticsearch at Kibana startup before
 retrying.
 
-`elasticsearch.url:`:: *Default: "http://localhost:9200"* The URL of the Elasticsearch instance to use for all your
-queries.
+`elasticsearch.hosts:`:: *Default: "http://localhost:9200"* The URLs of the Elasticsearch instances to use for all your
+queries.  All nodes listed here must be on the same cluster.
 
 `elasticsearch.username:` and `elasticsearch.password:`:: If your Elasticsearch is protected with basic authentication,
 these settings provide the username and password that the Kibana server uses to perform maintenance on the Kibana index at

packages/kbn-plugin-generator/README.md

@@ -64,7 +64,7 @@ Generated plugins receive a handful of scripts that can be used during developme
     Start kibana and have it include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
 
       ```
-      yarn start --elasticsearch.url http://localhost:9220
+      yarn start --elasticsearch.hosts http://localhost:9220
       ```
 
   - `yarn build`

packages/kbn-plugin-generator/sao_template/template/README.md

@@ -21,7 +21,7 @@ See the [kibana contributing guide](https://github.com/elastic/kibana/blob/maste
     Start kibana and have it include this plugin. You can pass any arguments that you would normally send to `bin/kibana`
 
       ```
-      yarn start --elasticsearch.url http://localhost:9220
+      yarn start --elasticsearch.hosts http://localhost:9220
       ```
 
   - `yarn build`

src/cli/serve/serve.js

@@ -105,7 +105,7 @@ function applyConfigOverrides(rawConfig, opts, extraCliOptions) {
     }
   }
 
-  if (opts.elasticsearch) set('elasticsearch.url', opts.elasticsearch);
+  if (opts.elasticsearch) set('elasticsearch.hosts', opts.elasticsearch.split(','));
   if (opts.port) set('server.port', opts.port);
   if (opts.host) set('server.host', opts.host);
   if (opts.quiet) set('logging.quiet', true);
@@ -144,7 +144,7 @@ export default function (program) {
   command
     .description('Run the kibana server')
     .collectUnknownOptions()
-    .option('-e, --elasticsearch <uri>', 'Elasticsearch instance')
+    .option('-e, --elasticsearch <uri1,uri2>', 'Elasticsearch instances')
     .option(
       '-c, --config <path>',
       'Path to the config file, can be changed with the CONFIG_PATH environment variable as well. ' +

src/legacy/core_plugins/console/index.js

@@ -18,11 +18,13 @@
  */
 
 import Boom from 'boom';
-import { resolveApi } from './api_server/server';
 import { resolve, join, sep } from 'path';
-import { has, isEmpty } from 'lodash';
-import setHeaders from '../elasticsearch/lib/set_headers';
+import url from 'url';
+import { has, isEmpty, head } from 'lodash';
+
+import { resolveApi } from './api_server/server';
 import { addExtensionSpecFilePath } from './api_server/spec';
+import setHeaders from '../elasticsearch/lib/set_headers';
 
 import {
   ProxyConfigCollection,
@@ -37,7 +39,7 @@ export default function (kibana) {
   const apps = [];
   return new kibana.Plugin({
     id: 'console',
-    require: [ 'elasticsearch' ],
+    require: ['elasticsearch'],
 
     config: function (Joi) {
       return Joi.object({
@@ -89,7 +91,7 @@ export default function (kibana) {
       const proxyPathFilters = options.proxyFilter.map(str => new RegExp(str));
 
       server.route(createProxyRoute({
-        baseUrl: config.get('elasticsearch.url'),
+        baseUrl: head(config.get('elasticsearch.hosts')),
         pathFilters: proxyPathFilters,
         getConfigForReq(req, uri) {
           const whitelist = config.get('elasticsearch.requestHeadersWhitelist');
@@ -132,7 +134,16 @@ export default function (kibana) {
 
       injectDefaultVars(server) {
         return {
-          elasticsearchUrl: server.config().get('elasticsearch.url')
+          elasticsearchUrl: url.format(
+            Object.assign(
+              url.parse(
+                head(
+                  server.config().get('elasticsearch.hosts')
+                )
+              ),
+              { auth: false }
+            )
+          )
         };
       },
 

src/legacy/core_plugins/console/public/src/sense_editor/editor.js

@@ -25,7 +25,7 @@ const RowParser = require('./row_parser');
 const InputMode = require('./mode/input');
 const utils = require('../utils');
 const es = require('../es');
-import chrome from 'ui/chrome';
+const chrome = require('ui/chrome');
 
 const smartResize = require('../smart_resize');
 
@@ -564,6 +564,7 @@ export default function SenseEditor($el) {
         const  esMethod = req.method;
         const  esData = req.data;
 
+        // this is the first url defined in elasticsearch.hosts
         const elasticsearchBaseUrl = chrome.getInjected('elasticsearchUrl');
         const url = es.constructESUrl(elasticsearchBaseUrl, esPath);
 

src/legacy/core_plugins/console/server/__tests__/elasticsearch_proxy_config.js

@@ -42,7 +42,7 @@ describe('plugins/console', function () {
         }
       };
 
-      server.config().get.withArgs('elasticsearch.url').returns('http://localhost:9200');
+      server.config().get.withArgs('elasticsearch.hosts').returns(['http://localhost:9200']);
       server.config().get.withArgs('elasticsearch.ssl.verificationMode').returns('full');
     });
 
@@ -58,20 +58,20 @@ describe('plugins/console', function () {
     });
 
     it(`uses https.Agent when url's protocol is https`, function () {
-      setElasticsearchConfig('url', 'https://localhost:9200');
+      setElasticsearchConfig('hosts', ['https://localhost:9200']);
       const { agent } = getElasticsearchProxyConfig(server);
       expect(agent).to.be.a(https.Agent);
     });
 
     it(`uses http.Agent when url's protocol is http`, function () {
-      setElasticsearchConfig('url', 'http://localhost:9200');
+      setElasticsearchConfig('hosts', ['http://localhost:9200']);
       const { agent } = getElasticsearchProxyConfig(server);
       expect(agent).to.be.a(http.Agent);
     });
 
     describe('ssl', function () {
       beforeEach(function () {
-        setElasticsearchConfig('url', 'https://localhost:9200');
+        setElasticsearchConfig('hosts', ['https://localhost:9200']);
       });
 
       it('sets rejectUnauthorized to false when verificationMode is none', function () {

src/legacy/core_plugins/console/server/elasticsearch_proxy_config.js

@@ -27,8 +27,9 @@ const readFile = (file) => readFileSync(file, 'utf8');
 
 const createAgent = (server) => {
   const config = server.config();
-  const target = url.parse(config.get('elasticsearch.url'));
-
+  const target = url.parse(
+    _.head(config.get('elasticsearch.hosts'))
+  );
   if (!/^https/.test(target.protocol)) return new http.Agent();
 
   const agentOptions = {};

src/legacy/core_plugins/elasticsearch/index.js

@@ -25,11 +25,10 @@ import { createDataCluster } from './lib/create_data_cluster';
 import { createAdminCluster } from './lib/create_admin_cluster';
 import { clientLogger } from './lib/client_logger';
 import { createClusters } from './lib/create_clusters';
-import filterHeaders from './lib/filter_headers';
-
 import { createProxy } from './lib/create_proxy';
+import filterHeaders from './lib/filter_headers';
 
-const DEFAULT_REQUEST_HEADERS = [ 'authorization' ];
+const DEFAULT_REQUEST_HEADERS = ['authorization'];
 
 export default function (kibana) {
   return new kibana.Plugin({
@@ -46,7 +45,10 @@ export default function (kibana) {
 
       return Joi.object({
         enabled: Joi.boolean().default(true),
-        url: Joi.string().uri({ scheme: ['http', 'https'] }).default('http://localhost:9200'),
+        sniffOnStart: Joi.boolean().default(false),
+        sniffInterval: Joi.number().allow(false).default(false),
+        sniffOnConnectionFault: Joi.boolean().default(false),
+        hosts: Joi.array().items(Joi.string().uri({ scheme: ['http', 'https'] })).single().default('http://localhost:9200'),
         preserveHost: Joi.boolean().default(true),
         username: Joi.string(),
         password: Joi.string(),
@@ -86,9 +88,27 @@ export default function (kibana) {
         };
       };
 
+      const url = () => {
+        return (settings, log) => {
+          const deprecatedUrl = get(settings, 'url');
+          const hosts = get(settings, 'hosts.length');
+          if (!deprecatedUrl) {
+            return;
+          }
+          if (hosts) {
+            log('Deprecated config key "elasticsearch.url" conflicts with "elasticsearch.hosts".  Ignoring "elasticsearch.url"');
+          } else {
+            set(settings, 'hosts', [deprecatedUrl]);
+            log('Config key "elasticsearch.url" is deprecated. It has been replaced with "elasticsearch.hosts"');
+          }
+          unset(settings, 'url');
+        };
+      };
+
       return [
         rename('ssl.ca', 'ssl.certificateAuthorities'),
         rename('ssl.cert', 'ssl.certificate'),
+        url(),
         sslVerify(),
       ];
     },
@@ -115,8 +135,7 @@ export default function (kibana) {
       createDataCluster(server);
       createAdminCluster(server);
 
-      createProxy(server, 'POST', '/{index}/_search');
-      createProxy(server, 'POST', '/_msearch');
+      createProxy(server);
 
       // Set up the health check service and start it.
       const { start, waitUntilReady } = healthCheck(this, server);