Merge branch 'master' of github.com:elastic/kibana into threshold-exc…

…eptions

.teamcity/src/builds/test/QuickTests.kt

@@ -13,8 +13,6 @@ object QuickTests : BuildType({
 
   val testScripts = mapOf(
     "Test Hardening" to ".ci/teamcity/tests/test_hardening.sh",
-    "X-Pack List cyclic dependency" to ".ci/teamcity/tests/xpack_list_cyclic_dependency.sh",
-    "X-Pack SIEM cyclic dependency" to ".ci/teamcity/tests/xpack_siem_cyclic_dependency.sh",
     "Test Projects" to ".ci/teamcity/tests/test_projects.sh",
     "Mocha Tests" to ".ci/teamcity/tests/mocha.sh"
   )

docs/developer/getting-started/debugging.asciidoc

@@ -15,7 +15,17 @@ For information about how to debug unit tests, refer to <<debugging-unit-tests>>
 https://github.com/elastic/apm-agent-nodejs[Elastic APM Node.js Agent]
 built-in for debugging purposes.
 
-Its default configuration is meant to be used by core {kib} developers
+With an application as varied and complex as Kibana has become, it's not practical or scalable to craft all possible performance measurements by hand ahead of time. As such, we need to rely on tooling to help us catch things we may otherwise have missed.
+
+For example, say you implement a brand new feature, plugin or service but don't quite know how it will impact Kibana's performance as a whole. APM allows us to not only spot that something is slow, but also hints at why it might be performing slowly. For example, if a function is slow on specific types of inputs, we can see where the time is spent by viewing the trace for that function call in the APM UI.
+
+image::images/apm_example_trace.png[]
+
+The net of metrics captured by APM are both a wide and deep because the entire application is instrumented at runtime and we simply take a sample of these metrics. This means that we don't have to know what we need to measure ahead of time, we'll instead just get (most) of the data we're likely going to need by default.
+
+This type of data can help us identify unknown bottlenecks, spot when a performance regression may have been introduced, and inform how the performance of Kibana is changing between releases. Using APM allows us to be proactive in getting ahead of potential performance regressions before they are released.
+
+The default APM configuration is meant to be used by core {kib} developers
 only, but it can easily be re-configured to your needs. In its default
 configuration it’s disabled and will, once enabled, send APM data to a
 centrally managed {es} cluster accessible only to Elastic
@@ -27,11 +37,8 @@ APM config option. To activate the APM agent, use the
 https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#active[`active`]
 APM config option.
 
-All config options can be set either via environment variables, or by
-creating an appropriate config file under `config/apm.dev.js`. For
-more information about configuring the APM agent, please refer to
-https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuring-the-agent.html[the
-documentation].
+All config options can be set by
+creating an appropriate config file under `config/apm.dev.js`. 
 
 Example `config/apm.dev.js` file:
 
@@ -56,4 +63,70 @@ ELASTIC_APM_ACTIVE=true yarn start
 Once the agent is active, it will trace all incoming HTTP requests to
 {kib}, monitor for errors, and collect process-level metrics. The
 collected data will be sent to the APM Server and is viewable in the APM
-UI in {kib}.
+UI in {kib}.
+
+[discrete]
+=== Running Kibana with the APM Agent Locally
+
+The easiest and recommended way of running Kibana with the APM agent locally is to use the solution provided by the https://github.com/elastic/apm-integration-testing[apm-integration-testing] repo. You’ll need https://www.docker.com/community-edition[Docker] and https://docs.docker.com/compose/install/[Docker Compose] to use the tool.
+
+[discrete]
+==== Quick start guide
+
+. Clone the https://github.com/elastic/apm-integration-testing[elastic/apm-integration-testing] repo.
+. Change into the apm-integration-testing repo: 
++
+[source,bash]
+----
+cd apm-integration-testing
+----
+
+. Run {es} and the APM servers without running Kibana:
++
+[source,bash]
+----
+./scripts/compose.py start master --no-kibana
+----
+
+. Change into the {kib} repo:
++
+[source,bash]
+----
+cd ../kibana
+----
+
+. Change the elasticsearch credentials in your `kibana.yml` configuration file to match those needed by elasticsearch and the APM server (see the apm-integration-testing repo's https://github.com/elastic/apm-integration-testing#logging-in[README] for users provided to test different scenarios).
+. Make sure that the APM agent is active and points to the local APM server by adding the following configuration settings to to a config file under `config/apm.dev.js`:
++
+Example `config/apm.dev.js` file:
++
+[source,js]
+----
+module.exports = {
+  active: true,
+  serverUrl: 'http://127.0.0.1:8200', // supports `http://localhost:8200`
+  centralConfig: false,
+  breakdownMetrics: false,
+  transactionSampleRate: 0.1,
+  metricsInterval: '120s'
+};
+----
+
+. Start Kibana with APM active using: 
++
+[source,bash]
+----
+yarn start
+----
+
+. After Kibana starts up, navigate to the APM app, where you should see some transactions.
+
+image::images/apm_ui_transactions.png[]
+
+You can now continue doing what you want to in Kibana (e.g. install sample data sets, issue queries in dashboards, build new visualizations etc).
+Once you're finished, you can stop Kibana normally, then stop the {es} and APM servers in the apm-integration-testing clone with the following script:
+
+[source,bash]
+----
+./scripts/compose.py stop
+----

docs/fleet/fleet.asciidoc

@@ -17,13 +17,11 @@ Standalone mode requires you to manually configure and manage the agent locally.
 * An overview of the data ingest in your {es} cluster.
 * Multiple integrations to collect and transform data.
 
-//TODO: Redo screen capture.
-
 [role="screenshot"]
 image::fleet/images/fleet-start.png[{fleet} app in {kib}]
 
 [float]
 == Get started
 
 To get started with {fleet}, refer to the
-{ingest-guide}/index.html[Ingest Management Guide].
+{ingest-guide}/index.html[{fleet}] docs.

docs/settings/fleet-settings.asciidoc

@@ -10,7 +10,7 @@ experimental[]
 You can configure `xpack.fleet` settings in your `kibana.yml`. 
 By default, {fleet} is enabled. To use {fleet}, you also need to configure {kib} and {es} hosts.
 
-See the {ingest-guide}/index.html[Ingest Management] docs for more information.
+See the {ingest-guide}/index.html[{fleet}] docs for more information.
 
 [[general-fleet-settings-kb]]
 ==== General {fleet} settings

package.json

@@ -731,7 +731,6 @@
     "loader-utils": "^1.2.3",
     "log-symbols": "^2.2.0",
     "lz-string": "^1.4.4",
-    "madge": "3.4.4",
     "mapbox-gl": "^1.12.0",
     "mapbox-gl-draw-rectangle-mode": "^1.0.4",
     "marge": "^1.0.1",

packages/kbn-es-archiver/src/cli.ts

@@ -26,8 +26,9 @@
 import Path from 'path';
 import Url from 'url';
 import readline from 'readline';
+import Fs from 'fs';
 
-import { RunWithCommands, createFlagError } from '@kbn/dev-utils';
+import { RunWithCommands, createFlagError, KbnClient, CA_CERT_PATH } from '@kbn/dev-utils';
 import { readConfigFile } from '@kbn/test';
 import legacyElasticsearch from 'elasticsearch';
 
@@ -40,13 +41,15 @@ export function runCli() {
   new RunWithCommands({
     description: 'CLI to manage archiving/restoring data in elasticsearch',
     globalFlags: {
-      string: ['es-url', 'kibana-url', 'dir', 'config'],
+      string: ['es-url', 'kibana-url', 'dir', 'config', 'es-ca', 'kibana-ca'],
       help: `
         --config           path to an FTR config file that sets --es-url, --kibana-url, and --dir
                              default: ${defaultConfigPath}
         --es-url           url for Elasticsearch, prefer the --config flag
         --kibana-url       url for Kibana, prefer the --config flag
         --dir              where arechives are stored, prefer the --config flag
+        --kibana-ca        if Kibana url points to https://localhost we default to the CA from @kbn/dev-utils, customize the CA with this flag
+        --es-ca            if Elasticsearch url points to https://localhost we default to the CA from @kbn/dev-utils, customize the CA with this flag
       `,
     },
     async extendContext({ log, flags, addCleanupTask }) {
@@ -78,6 +81,40 @@ export function runCli() {
         throw createFlagError('--kibana-url or --config must be defined');
       }
 
+      const kibanaCaPath = flags['kibana-ca'];
+      if (kibanaCaPath && typeof kibanaCaPath !== 'string') {
+        throw createFlagError('--kibana-ca must be a string');
+      }
+
+      let kibanaCa;
+      if (config.get('servers.kibana.certificateAuthorities') && !kibanaCaPath) {
+        kibanaCa = config.get('servers.kibana.certificateAuthorities');
+      } else if (kibanaCaPath) {
+        kibanaCa = Fs.readFileSync(kibanaCaPath);
+      } else {
+        const { protocol, hostname } = Url.parse(kibanaUrl);
+        if (protocol === 'https:' && hostname === 'localhost') {
+          kibanaCa = Fs.readFileSync(CA_CERT_PATH);
+        }
+      }
+
+      const esCaPath = flags['es-ca'];
+      if (esCaPath && typeof esCaPath !== 'string') {
+        throw createFlagError('--es-ca must be a string');
+      }
+
+      let esCa;
+      if (config.get('servers.elasticsearch.certificateAuthorities') && !esCaPath) {
+        esCa = config.get('servers.elasticsearch.certificateAuthorities');
+      } else if (esCaPath) {
+        esCa = Fs.readFileSync(esCaPath);
+      } else {
+        const { protocol, hostname } = Url.parse(kibanaUrl);
+        if (protocol === 'https:' && hostname === 'localhost') {
+          esCa = Fs.readFileSync(CA_CERT_PATH);
+        }
+      }
+
       let dir = flags.dir;
       if (dir && typeof dir !== 'string') {
         throw createFlagError('--dir must be a string');
@@ -91,15 +128,22 @@ export function runCli() {
 
       const client = new legacyElasticsearch.Client({
         host: esUrl,
+        ssl: esCa ? { ca: esCa } : undefined,
         log: flags.verbose ? 'trace' : [],
       });
       addCleanupTask(() => client.close());
 
+      const kbnClient = new KbnClient({
+        log,
+        url: kibanaUrl,
+        certificateAuthorities: kibanaCa ? [kibanaCa] : undefined,
+      });
+
       const esArchiver = new EsArchiver({
         log,
         client,
         dataDir: dir,
-        kibanaUrl,
+        kbnClient,
       });
 
       return {

packages/kbn-es-archiver/src/es_archiver.ts

@@ -29,27 +29,24 @@ import {
   editAction,
 } from './actions';
 
+interface Options {
+  client: Client;
+  dataDir: string;
+  log: ToolingLog;
+  kbnClient: KbnClient;
+}
+
 export class EsArchiver {
   private readonly client: Client;
   private readonly dataDir: string;
   private readonly log: ToolingLog;
   private readonly kbnClient: KbnClient;
 
-  constructor({
-    client,
-    dataDir,
-    log,
-    kibanaUrl,
-  }: {
-    client: Client;
-    dataDir: string;
-    log: ToolingLog;
-    kibanaUrl: string;
-  }) {
-    this.client = client;
-    this.dataDir = dataDir;
-    this.log = log;
-    this.kbnClient = new KbnClient({ log, url: kibanaUrl });
+  constructor(options: Options) {
+    this.client = options.client;
+    this.dataDir = options.dataDir;
+    this.log = options.log;
+    this.kbnClient = options.kbnClient;
   }
 
   /**

test/common/services/es_archiver.ts

@@ -17,17 +17,18 @@
  * under the License.
  */
 
-import { format as formatUrl } from 'url';
 import { EsArchiver } from '@kbn/es-archiver';
 import { FtrProviderContext } from '../ftr_provider_context';
 
 // @ts-ignore not TS yet
 import * as KibanaServer from './kibana_server';
 
-export function EsArchiverProvider({ getService, hasService }: FtrProviderContext): EsArchiver {
+export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver {
   const config = getService('config');
   const client = getService('legacyEs');
   const log = getService('log');
+  const kibanaServer = getService('kibanaServer');
+  const retry = getService('retry');
 
   if (!config.get('esArchiver')) {
     throw new Error(`esArchiver can't be used unless you specify it's config in your config file`);
@@ -39,17 +40,15 @@ export function EsArchiverProvider({ getService, hasService }: FtrProviderContex
     client,
     dataDir,
     log,
-    kibanaUrl: formatUrl(config.get('servers.kibana')),
+    kbnClient: kibanaServer,
   });
 
-  if (hasService('kibanaServer')) {
-    KibanaServer.extendEsArchiver({
-      esArchiver,
-      kibanaServer: getService('kibanaServer'),
-      retry: getService('retry'),
-      defaults: config.get('uiSettings.defaults'),
-    });
-  }
+  KibanaServer.extendEsArchiver({
+    esArchiver,
+    kibanaServer,
+    retry,
+    defaults: config.get('uiSettings.defaults'),
+  });
 
   return esArchiver;
 }

test/scripts/jenkins_xpack.sh

@@ -9,18 +9,6 @@ if [[ -z "$CODE_COVERAGE" ]] ; then
   echo ""
   echo ""
 
-  echo " -> Running Security Solution cyclic dependency test"
-  cd "$XPACK_DIR"
-  checks-reporter-with-killswitch "X-Pack Security Solution cyclic dependency test" node plugins/security_solution/scripts/check_circular_deps
-  echo ""
-  echo ""
-
-  echo " -> Running List cyclic dependency test"
-  cd "$XPACK_DIR"
-  checks-reporter-with-killswitch "X-Pack List cyclic dependency test" node plugins/lists/scripts/check_circular_deps
-  echo ""
-  echo ""
-
   # echo " -> Running jest integration tests"
   # cd "$XPACK_DIR"
   # node scripts/jest_integration --ci --verbose

vars/tasks.groovy

@@ -36,8 +36,6 @@ def test() {
 
     kibanaPipeline.scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh'),
     kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh'),
-    kibanaPipeline.scriptTask('X-Pack SIEM cyclic dependency', 'test/scripts/test/xpack_siem_cyclic_dependency.sh'),
-    kibanaPipeline.scriptTask('X-Pack List cyclic dependency', 'test/scripts/test/xpack_list_cyclic_dependency.sh'),
     kibanaPipeline.scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh'),
   ])
 }

x-pack/plugins/fleet/common/constants/epm.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
+export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;