Merge branch 'master' of github.com:elastic/kibana into dev/improve-s…

…tate-sync-error-handling

.ci/Dockerfile

@@ -0,0 +1,38 @@
+# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
+# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
+
+ARG NODE_VERSION=10.21.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana

.ci/Jenkinsfile_baseline_capture

@@ -7,18 +7,22 @@ kibanaPipeline(timeoutMinutes: 120) {
   githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') {
     ciStats.trackBuild {
       catchError {
-        parallel([
-          'oss-visualRegression': {
-            workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
-            }
-          },
-          'xpack-visualRegression': {
-            workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
-            }
-          },
-        ])
+        withEnv([
+          'CI_PARALLEL_PROCESS_NUMBER=1'
+        ]) {
+          parallel([
+            'oss-visualRegression': {
+              workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')()
+              }
+            },
+            'xpack-visualRegression': {
+              workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')()
+              }
+            },
+          ])
+        }
       }
 
       kibanaPipeline.sendMail()

.ci/runbld_no_junit.yml

@@ -3,4 +3,4 @@
 profiles:
 - ".*": # Match any job
     tests:
-      junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
+      junit-filename-pattern: false

.gitignore

@@ -49,6 +49,8 @@ npm-debug.log*
 .tern-project
 .nyc_output
 .ci/pipeline-library/build/
+.ci/runbld
+.ci/bash_standard_lib.sh
 .gradle
 
 # apm plugin

Jenkinsfile

@@ -9,49 +9,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true)
       ciStats.trackBuild {
         catchError {
           retryable.enable()
-          parallel([
-            'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-            'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-            'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-              'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-              'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-              'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-              'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-              'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-              'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-              'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-              'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-              'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-              'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-              'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-              'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-              'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-              'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-              // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-            ]),
-            'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-              'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-              'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-              'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-              'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-              'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-              'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-              'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-              'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-              'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-              'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-              'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-              'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-              'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'),
-              'xpack-securitySolutionCypress': { processNumber ->
-                whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/', 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/', 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx']) {
-                  kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
-                }
-              },
-
-              // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-            ]),
-          ])
+          kibanaPipeline.allCiTasks()
         }
       }
     }

docs/observability/index.asciidoc

@@ -0,0 +1,24 @@
+[chapter]
+[role="xpack"]
+[[observability]]
+= Observability
+
+Observability enables you to add and monitor your logs, system
+metrics, uptime data, and application traces, as a single stack. 
+
+With *Observability*, you have:
+
+* A central place to add and configure your data sources.
+* A variety of charts displaying analytics relating to each data source.
+* *View in app* options to drill down and analyze data in the Logs, Metrics, Uptime, and APM apps.
+* An alerts chart to keep you informed of any issues that you may need to resolve quickly.
+
+[role="screenshot"]
+image::observability/images/observability-overview.png[Observability Overview in {kib}]
+
+[float]
+== Get started
+
+{kib} provides step-by-step instructions to help you add and configure your data
+sources. The {observability-guide}/index.html[Observability Guide] is a good source for more detailed information
+and instructions.

docs/user/index.asciidoc

@@ -27,6 +27,8 @@ include::graph/index.asciidoc[]
 
 include::visualize.asciidoc[]
 
+include::{kib-repo-dir}/observability/index.asciidoc[]
+
 include::{kib-repo-dir}/logs/index.asciidoc[]
 
 include::{kib-repo-dir}/infrastructure/index.asciidoc[]

packages/kbn-dev-utils/src/run/help.test.ts

@@ -57,7 +57,7 @@ const barCommand: Command<any> = {
   usage: 'bar [...names]',
 };
 
-describe('getHelp()', () => {
+describe.skip('getHelp()', () => {
   it('returns the expected output', () => {
     expect(
       getHelp({
@@ -95,7 +95,7 @@ describe('getHelp()', () => {
   });
 });
 
-describe('getCommandLevelHelp()', () => {
+describe.skip('getCommandLevelHelp()', () => {
   it('returns the expected output', () => {
     expect(
       getCommandLevelHelp({
@@ -141,7 +141,7 @@ describe('getCommandLevelHelp()', () => {
   });
 });
 
-describe('getHelpForAllCommands()', () => {
+describe.skip('getHelpForAllCommands()', () => {
   it('returns the expected output', () => {
     expect(
       getHelpForAllCommands({

packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts

@@ -19,9 +19,12 @@
 
 import { REPO_ROOT } from '../repo_root';
 
-export function createAbsolutePathSerializer(rootPath: string = REPO_ROOT) {
+export function createAbsolutePathSerializer(
+  rootPath: string = REPO_ROOT,
+  replacement = '<absolute path>'
+) {
   return {
     test: (value: any) => typeof value === 'string' && value.startsWith(rootPath),
-    serialize: (value: string) => value.replace(rootPath, '<absolute path>').replace(/\\/g, '/'),
+    serialize: (value: string) => value.replace(rootPath, replacement).replace(/\\/g, '/'),
   };
 }

packages/kbn-optimizer/package.json

@@ -15,12 +15,9 @@
     "@kbn/dev-utils": "1.0.0",
     "@kbn/ui-shared-deps": "1.0.0",
     "@types/compression-webpack-plugin": "^2.0.2",
-    "@types/estree": "^0.0.44",
     "@types/loader-utils": "^1.1.3",
     "@types/watchpack": "^1.1.5",
     "@types/webpack": "^4.41.3",
-    "acorn": "^7.1.1",
-    "acorn-walk": "^7.1.1",
     "autoprefixer": "^9.7.4",
     "babel-loader": "^8.0.6",
     "clean-webpack-plugin": "^3.0.0",

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -104,4 +104,18 @@ export class BundleCache {
   public getOptimizerCacheKey() {
     return this.get().optimizerCacheKey;
   }
+
+  public clear() {
+    this.state = undefined;
+
+    if (this.path) {
+      try {
+        Fs.unlinkSync(this.path);
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+    }
+  }
 }

packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts

@@ -21,7 +21,9 @@ import { createAbsolutePathSerializer } from '@kbn/dev-utils';
 
 import { getPluginBundles } from './get_plugin_bundles';
 
-expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '<repoRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '<outputRoot>'));
+expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '<outsideOfRepo>'));
 
 it('returns a bundle for core and each plugin', () => {
   expect(
@@ -56,46 +58,47 @@ it('returns a bundle for core and each plugin', () => {
           manifestPath: '/repo/x-pack/plugins/box/kibana.json',
         },
       ],
-      '/repo'
+      '/repo',
+      '/output'
     ).map((b) => b.toSpec())
   ).toMatchInlineSnapshot(`
     Array [
       Object {
         "banner": undefined,
-        "contextDir": <absolute path>/plugins/foo,
+        "contextDir": <repoRoot>/plugins/foo,
         "id": "foo",
-        "manifestPath": <absolute path>/plugins/foo/kibana.json,
-        "outputDir": <absolute path>/plugins/foo/target/public,
+        "manifestPath": <repoRoot>/plugins/foo/kibana.json,
+        "outputDir": <outputRoot>/plugins/foo/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
       Object {
         "banner": undefined,
-        "contextDir": "/outside/of/repo/plugins/baz",
+        "contextDir": <outsideOfRepo>/plugins/baz,
         "id": "baz",
-        "manifestPath": "/outside/of/repo/plugins/baz/kibana.json",
-        "outputDir": "/outside/of/repo/plugins/baz/target/public",
+        "manifestPath": <outsideOfRepo>/plugins/baz/kibana.json,
+        "outputDir": <outsideOfRepo>/plugins/baz/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
       Object {
         "banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.
      * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */
     ",
-        "contextDir": <absolute path>/x-pack/plugins/box,
+        "contextDir": <repoRoot>/x-pack/plugins/box,
         "id": "box",
-        "manifestPath": <absolute path>/x-pack/plugins/box/kibana.json,
-        "outputDir": <absolute path>/x-pack/plugins/box/target/public,
+        "manifestPath": <repoRoot>/x-pack/plugins/box/kibana.json,
+        "outputDir": <outputRoot>/x-pack/plugins/box/target/public,
         "publicDirNames": Array [
           "public",
         ],
-        "sourceRoot": <absolute path>,
+        "sourceRoot": <repoRoot>,
         "type": "plugin",
       },
     ]

packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts

@@ -23,7 +23,11 @@ import { Bundle } from '../common';
 
 import { KibanaPlatformPlugin } from './kibana_platform_plugins';
 
-export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: string) {
+export function getPluginBundles(
+  plugins: KibanaPlatformPlugin[],
+  repoRoot: string,
+  outputRoot: string
+) {
   const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep;
 
   return plugins
@@ -36,7 +40,11 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri
           publicDirNames: ['public', ...p.extraPublicDirs],
           sourceRoot: repoRoot,
           contextDir: p.directory,
-          outputDir: Path.resolve(p.directory, 'target/public'),
+          outputDir: Path.resolve(
+            outputRoot,
+            Path.relative(repoRoot, p.directory),
+            'target/public'
+          ),
           manifestPath: p.manifestPath,
           banner: p.directory.startsWith(xpackDirSlash)
             ? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.\n` +