Merge branch 'master' of github.com:elastic/kibana into remote_cluster

.backportrc.json

@@ -3,6 +3,7 @@
   "targetBranchChoices": [
     { "name": "master", "checked": true },
     { "name": "7.x", "checked": true },
+    "7.9",
     "7.8",
     "7.7",
     "7.6",
@@ -26,7 +27,7 @@
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.0.0$": "master",
-    "^v7.9.0$": "7.x",
+    "^v7.10.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }
 }

.browserslistrc

@@ -1,7 +1,9 @@
 [production]
-last 2 versions
-> 5%
-Safari 7 # for PhantomJS support: https://github.com/elastic/kibana/issues/27136
+last 2 Firefox versions
+last 2 Chrome versions
+last 2 Safari versions
+> 0.25%
+not ie 11
 
 [dev]
 last 1 chrome versions

.ci/Dockerfile

@@ -0,0 +1,38 @@
+# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
+# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
+
+ARG NODE_VERSION=10.22.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana

.ci/Jenkinsfile_baseline_capture

@@ -0,0 +1,32 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+
+kibanaPipeline(timeoutMinutes: 120) {
+  githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') {
+    ciStats.trackBuild {
+      catchError {
+        withEnv([
+          'CI_PARALLEL_PROCESS_NUMBER=1'
+        ]) {
+          parallel([
+            'oss-visualRegression': {
+              workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')()
+              }
+            },
+            'xpack-visualRegression': {
+              workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')()
+              }
+            },
+          ])
+        }
+      }
+
+      kibanaPipeline.sendMail()
+      slackNotifications.onFailure()
+    }
+  }
+}

.ci/Jenkinsfile_baseline_trigger

@@ -0,0 +1,70 @@
+#!/bin/groovy
+
+def MAXIMUM_COMMITS_TO_CHECK = 10
+def MAXIMUM_COMMITS_TO_BUILD = 5
+
+if (!params.branches_yaml) {
+  error "'branches_yaml' parameter must be specified"
+}
+
+def additionalBranches = []
+
+def branches = readYaml(text: params.branches_yaml) + additionalBranches
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+
+withGithubCredentials {
+  branches.each { branch ->
+    if (branch == '6.8') {
+      // skip 6.8, it is tracked but we don't need snapshots for it and haven't backported
+      // the baseline capture scripts to it.
+      return;
+    }
+
+    stage(branch) {
+      def commits = getCommits(branch, MAXIMUM_COMMITS_TO_CHECK, MAXIMUM_COMMITS_TO_BUILD)
+
+      commits.take(MAXIMUM_COMMITS_TO_BUILD).each { commit ->
+        catchErrors {
+          githubCommitStatus.create(commit, 'pending', 'Baseline started.', 'kibana-ci-baseline')
+
+          build(
+            propagate: false,
+            wait: false,
+            job: 'elastic+kibana+baseline-capture',
+            parameters: [
+              string(name: 'branch_specifier', value: branch),
+              string(name: 'commit', value: commit),
+            ]
+          )
+        }
+      }
+    }
+  }
+}
+
+def getCommits(String branch, maximumCommitsToCheck, maximumCommitsToBuild) {
+  print "Getting latest commits for ${branch}..."
+  def commits = githubApi.get("repos/elastic/kibana/commits?sha=${branch}").take(maximumCommitsToCheck).collect { it.sha }
+  def commitsToBuild = []
+
+  for (commit in commits) {
+    print "Getting statuses for ${commit}"
+    def status = githubApi.get("repos/elastic/kibana/statuses/${commit}").find { it.context == 'kibana-ci-baseline' }
+    print "Commit '${commit}' already built? ${status ? 'Yes' : 'No'}"
+
+    if (!status) {
+      commitsToBuild << commit
+    } else {
+      // Stop at the first commit we find that's already been triggered
+      break
+    }
+
+    if (commitsToBuild.size() >= maximumCommitsToBuild) {
+      break
+    }
+  }
+
+  return commitsToBuild.reverse() // We want the builds to trigger oldest-to-newest
+}

.ci/Jenkinsfile_coverage

@@ -13,6 +13,7 @@ kibanaPipeline(timeoutMinutes: 240) {
       workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) {
         catchError {
           kibanaCoverage.runTests()
+          kibanaTeamAssign.load('team_assignment', "### Upload Team Assignment JSON")
           handleIngestion(TIME_STAMP)
         }
         handleFail()
@@ -23,18 +24,25 @@ kibanaPipeline(timeoutMinutes: 240) {
 }
 
 def handleIngestion(timestamp) {
+  def previousSha = handlePreviousSha()
   kibanaPipeline.downloadCoverageArtifacts()
   kibanaCoverage.prokLinks("### Process HTML Links")
   kibanaCoverage.collectVcsInfo("### Collect VCS Info")
   kibanaCoverage.generateReports("### Merge coverage reports")
   kibanaCoverage.uploadCombinedReports()
-  kibanaCoverage.ingest(timestamp, '### Injest && Upload')
+  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, '### Ingest && Upload')
   kibanaCoverage.uploadCoverageStaticSite(timestamp)
 }
 
+def handlePreviousSha() {
+  def previous = kibanaCoverage.downloadPrevious('### Download OLD Previous')
+  kibanaCoverage.uploadPrevious('### Upload NEW Previous')
+  return previous
+}
+
 def handleFail() {
   def buildStatus = buildUtils.getBuildStatus()
-  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED') {
+  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED' && buildStatus != 'UNSTABLE') {
     slackNotifications.sendFailedBuild(
       channel: '#kibana-qa',
       username: 'Kibana QA'

.ci/Jenkinsfile_security_cypress

@@ -0,0 +1,28 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+
+kibanaPipeline(timeoutMinutes: 180) {
+  slackNotifications.onFailure(
+    disabled: !params.NOTIFY_ON_FAILURE,
+    channel: '#security-solution-slack-testing'
+  ) {
+    catchError {
+      withEnv([
+        'CI_PARALLEL_PROCESS_NUMBER=1'
+      ]) {
+        def job = 'xpack-securityCypress'
+
+        workers.ci(name: job, size: 'l', ramDisk: true) {
+          kibanaPipeline.bash('test/scripts/jenkins_xpack_build_kibana.sh', 'Build Default Distributable')
+          kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress.sh')()
+        }
+      }
+    }
+  }
+
+  if (params.NOTIFY_ON_FAILURE) {
+    kibanaPipeline.sendMail(to: 'siem_dev_team@elastic.co')
+  }
+}

.ci/end2end.groovy

@@ -14,6 +14,7 @@ pipeline {
     HOME = "${env.WORKSPACE}"
     E2E_DIR = 'x-pack/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
+    KBN_OPTIMIZER_THEMES = 'v7light'
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -109,6 +110,9 @@ pipeline {
         archiveArtifacts(allowEmptyArchive: true, artifacts: "${E2E_DIR}/kibana.log")
       }
     }
+    cleanup {
+      notifyBuildResult(prComment: false, analyzeFlakey: false, shouldNotify: false)
+    }
   }
 }
 

.ci/es-snapshots/Jenkinsfile_build_es

@@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
 timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
-      node(workers.label('s')) {
+      node(workers.label('l')) {
         catchErrors {
           def VERSION
           def SNAPSHOT_ID
@@ -154,9 +154,10 @@ def buildArchives(destination) {
     "NODE_NAME=",
   ]) {
     sh """
-      ./gradlew -p distribution/archives assemble --parallel
+      ./gradlew -Dbuild.docker=true assemble --parallel
       mkdir -p ${destination}
-      find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
+      find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
+      docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
     """
   }
 }

.ci/packer_cache_for_branch.sh

@@ -18,7 +18,7 @@ node scripts/es snapshot --download-only;
 node scripts/es snapshot --license=oss --download-only;
 
 # download reporting browsers
-(cd "x-pack" && yarn gulp prepare);
+(cd "x-pack" && yarn gulp downloadChromium);
 
 # cache the chromedriver archive
 chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")"
@@ -46,7 +46,7 @@ echo "Creating bootstrap_cache archive"
 # archive cacheable directories
 mkdir -p "$HOME/.kibana/bootstrap_cache"
 tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
-  x-pack/plugins/reporting/.chromium \
+  .chromium \
   .es \
   .chromedriver \
   .geckodriver;

.ci/pipeline-library/src/test/KibanaBasePipelineTest.groovy

@@ -78,6 +78,10 @@ class KibanaBasePipelineTest extends BasePipelineTest {
     return helper.callStack.find { it.methodName == name }
   }
 
+  def fnMocks(String name) {
+    helper.callStack.findAll { it.methodName == name }
+  }
+
   void mockFailureBuild() {
     props([
       buildUtils: [

.ci/pipeline-library/src/test/githubCommitStatus.groovy

@@ -12,6 +12,7 @@ class GithubCommitStatusTest extends KibanaBasePipelineTest {
 
   interface BuildState {
     Object get(String key)
+    Object has(String key)
   }
 
   interface GithubApi {
@@ -25,6 +26,7 @@ class GithubCommitStatusTest extends KibanaBasePipelineTest {
     buildStateMock = mock(BuildState)
     githubApiMock = mock(GithubApi)
 
+    when(buildStateMock.has('checkoutInfo')).thenReturn(true)
     when(buildStateMock.get('checkoutInfo')).thenReturn([ commit: 'COMMIT_HASH', ])
     when(githubApiMock.post(any(), any())).thenReturn(null)
 

.ci/pipeline-library/src/test/prChanges.groovy

@@ -84,4 +84,30 @@ class PrChangesTest extends KibanaBasePipelineTest {
 
     assertFalse(prChanges.areChangesSkippable())
   }
+
+  @Test
+  void 'areChangesSkippable() with skippable changes that are in notSkippablePaths'() {
+    props([
+      githubPrs: [
+        getChanges: { [
+          [filename: 'docs/developer/plugin-list.asciidoc'],
+        ] },
+      ],
+    ])
+
+    assertFalse(prChanges.areChangesSkippable())
+  }
+
+  @Test
+  void 'areChangesSkippable() with plugin readme changes'() {
+    props([
+      githubPrs: [
+        getChanges: { [
+          [filename: 'src/plugins/foo/README.asciidoc'],
+        ] },
+      ],
+    ])
+
+    assertFalse(prChanges.areChangesSkippable())
+  }
 }