Merge branch 'master' into data_view_share_capable

.backportrc.json

@@ -2,7 +2,7 @@
  "upstream": "elastic/kibana",
  "targetBranchChoices": [
  { "name": "master", "checked": true },
- { "name": "7.x", "checked": true },
+ "7.16",
  "7.15",
  "7.14",
  "7.13",
@@ -33,7 +33,6 @@
  "targetPRLabels": ["backport"],
  "branchLabelMapping": {
  "^v8.0.0$": "master",
- "^v7.16.0$": "7.x",
  "^v(\\d+).(\\d+).\\d+$": "$1.$2"
  },
  "autoMerge": true,

.bazeliskversion

@@ -1 +1 @@
-1.7.5
+1.10.1

.bazelrc

@@ -10,9 +10,6 @@ build --remote_timeout=30
 build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
 build --remote_accept_cached=true
 
-# BuildBuddy
-## Metadata settings
-build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.bazelrc.common

@@ -120,8 +120,8 @@ test --incompatible_strict_action_env
 # collect coverage information from test targets
 coverage --instrument_test_targets
 
-# Settings for CI
-# Bazel flags for CI are in /src/dev/ci_setup/.bazelrc-ci
+# Metadata settings
+build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members

.bazelversion

@@ -1 +1 @@
-4.0.0
+4.2.1

.buildkite/pipelines/es_snapshots/verify.yml

@@ -13,6 +13,7 @@ steps:
 
  - command: .buildkite/scripts/lifecycle/pre_build.sh
  label: Pre-Build
+ timeout_in_minutes: 10
 
  - wait
 
@@ -22,6 +23,7 @@ steps:
  queue: c2-16
  key: build
  if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+ timeout_in_minutes: 60
 
  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
  label: 'Default CI Group'
@@ -81,6 +83,7 @@ steps:
 
  - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
  label: Trigger promotion
+ timeout_in_minutes: 10
  depends_on:
  - default-cigroup
  - default-cigroup-docker
@@ -93,3 +96,4 @@ steps:
 
  - command: .buildkite/scripts/lifecycle/post_build.sh
  label: Post-Build
+ timeout_in_minutes: 10

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -17,12 +17,6 @@ const inputs = [
  default: 0,
  required: true,
  },
- {
- key: 'ftsr-concurrency',
- text: 'Max concurrency per step',
- default: 20,
- required: true,
- },
 ];
 
 for (let i = 1; i <= OSS_CI_GROUPS; i++) {
@@ -36,7 +30,7 @@ for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
 const pipeline = {
  steps: [
  {
- input: 'Number of Runs',
+ input: 'Number of Runs - Click Me',
  fields: inputs,
  },
  {

.buildkite/pipelines/flaky_tests/runner.js

@@ -9,8 +9,10 @@ const overrideCount = parseInt(
  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
 );
 
-const concurrency =
- parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20;
+const concurrency = 25;
+const initialJobs = 3;
+
+let totalJobs = initialJobs;
 
 const testSuites = [];
 for (const key of keys) {
@@ -21,12 +23,25 @@ for (const key of keys) {
  const value =
  overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
 
+ const count = value === '' ? defaultCount : parseInt(value);
+ totalJobs += count;
+
  testSuites.push({
  key: key.replace('ftsr-suite/', ''),
- count: value === '' ? defaultCount : parseInt(value),
+ count: count,
  });
 }
 
+if (totalJobs > 500) {
+ console.error('+++ Too many tests');
+ console.error(
+ `Buildkite builds can only contain 500 steps in total. Found ${totalJobs} in total. Make sure your test runs are less than ${
+ 500 - initialJobs
+ }`
+ );
+ process.exit(1);
+}
+
 const steps = [];
 const pipeline = {
  env: {
@@ -46,7 +61,7 @@ steps.push({
 for (const testSuite of testSuites) {
  const TEST_SUITE = testSuite.key;
  const RUN_COUNT = testSuite.count;
- const UUID = TEST_SUITE + process.env.UUID;
+ const UUID = process.env.UUID;
 
  const JOB_PARTS = TEST_SUITE.split('/');
  const IS_XPACK = JOB_PARTS[0] === 'xpack';
@@ -65,6 +80,7 @@ for (const testSuite of testSuites) {
  parallelism: RUN_COUNT,
  concurrency: concurrency,
  concurrency_group: UUID,
+ concurrency_method: 'eager',
  });
  } else {
  steps.push({
@@ -75,6 +91,7 @@ for (const testSuite of testSuites) {
  parallelism: RUN_COUNT,
  concurrency: concurrency,
  concurrency_group: UUID,
+ concurrency_method: 'eager',
  });
  }
 }

.buildkite/pipelines/hourly.yml

@@ -3,6 +3,7 @@ env:
 steps:
  - command: .buildkite/scripts/lifecycle/pre_build.sh
  label: Pre-Build
+ timeout_in_minutes: 10
 
  - wait
 
@@ -12,6 +13,7 @@ steps:
  queue: c2-16
  key: build
  if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+ timeout_in_minutes: 60
 
  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
  label: 'Default CI Group'
@@ -143,21 +145,25 @@ steps:
  agents:
  queue: n2-2
  key: linting
+ timeout_in_minutes: 90
 
  - command: .buildkite/scripts/steps/checks.sh
  label: 'Checks'
  agents:
  queue: c2-4
  key: checks
+ timeout_in_minutes: 120
 
  - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
  label: 'Build Storybooks'
  agents:
  queue: c2-4
  key: storybooks
+ timeout_in_minutes: 60
 
  - wait: ~
  continue_on_failure: true
 
  - command: .buildkite/scripts/lifecycle/post_build.sh
  label: Post-Build
+ timeout_in_minutes: 10

.buildkite/pipelines/on_merge.yml

@@ -4,21 +4,27 @@ env:
 steps:
  - command: .buildkite/scripts/lifecycle/pre_build.sh
  label: Pre-Build
+ timeout_in_minutes: 10
 
  - wait
 
  - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
  label: Default Build and Metrics
+ env:
+ BAZEL_CACHE_MODE: read-write
  agents:
  queue: c2-8
+ timeout_in_minutes: 60
 
  - command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
  label: Build TS Refs and Check Public API Docs
  agents:
  queue: c2-4
+ timeout_in_minutes: 80
 
  - wait: ~
  continue_on_failure: true
 
  - command: .buildkite/scripts/lifecycle/post_build.sh
  label: Post-Build
+ timeout_in_minutes: 10

.buildkite/pipelines/pull_request/base.yml

@@ -1,6 +1,7 @@
 steps:
  - command: .buildkite/scripts/lifecycle/pre_build.sh
  label: Pre-Build
+ timeout_in_minutes: 10
 
  - wait
 
@@ -10,14 +11,15 @@ steps:
  queue: c2-16
  key: build
  if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+ timeout_in_minutes: 60
 
  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
  label: 'Default CI Group'
  parallelism: 13
  agents:
  queue: ci-group-6
  depends_on: build
- timeout_in_minutes: 120
+ timeout_in_minutes: 150
  key: default-cigroup
  retry:
  automatic:
@@ -141,15 +143,18 @@ steps:
  agents:
  queue: n2-2
  key: linting
+ timeout_in_minutes: 90
 
  - command: .buildkite/scripts/steps/checks.sh
  label: 'Checks'
  agents:
  queue: c2-4
  key: checks
+ timeout_in_minutes: 120
 
  - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
  label: 'Build Storybooks'
  agents:
  queue: c2-4
  key: storybooks
+ timeout_in_minutes: 60

.buildkite/pipelines/pull_request/uptime.yml

@@ -0,0 +1,11 @@
+steps:
+ - command: .buildkite/scripts/steps/functional/uptime.sh
+ label: 'Uptime @elastic/synthetics Tests'
+ agents:
+ queue: ci-group-6
+ depends_on: build
+ timeout_in_minutes: 120
+ retry:
+ automatic:
+ - exit_status: '*'
+ limit: 1

.buildkite/pipelines/update_demo_env.yml

@@ -1,8 +1,10 @@
 steps:
  - command: .buildkite/scripts/steps/demo_env/es_and_init.sh
  label: Initialize Environment and Deploy ES
+ timeout_in_minutes: 10
 
  - command: .buildkite/scripts/steps/demo_env/kibana.sh
  label: Build and Deploy Kibana
  agents:
  queue: c2-8
+ timeout_in_minutes: 60

.buildkite/pull_requests.json

@@ -15,7 +15,8 @@
  "build_on_comment": true,
  "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
  "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
- "labels": ["buildkite-ci"]
+ "skip_ci_labels": ["skip-ci", "jenkins-ci"],
+ "skip_target_branches": ["6.8"]
  }
  ]
 }

.buildkite/scripts/bootstrap.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/common/setup_bazel.sh
 
 echo "--- yarn install and bootstrap"
 retry 2 15 yarn kbn bootstrap

.buildkite/scripts/common/setup_bazel.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+echo "[bazel] writing .bazelrc"
+cat <<EOF > $KIBANA_DIR/.bazelrc
+ # Generated by .buildkite/scripts/common/setup_bazel.sh
+
+ import %workspace%/.bazelrc.common
+
+ build --build_metadata=ROLE=CI
+EOF
+
+if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
+ echo "[bazel] enabling caching"
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+ build --bes_results_url=https://app.buildbuddy.io/invocation/
+ build --bes_backend=grpcs://cloud.buildbuddy.io
+ build --remote_cache=grpcs://cloud.buildbuddy.io
+ build --remote_timeout=3600
+ build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
+EOF
+fi
+
+if [[ "${BAZEL_CACHE_MODE:-none}" == "read" ]]; then
+ echo "[bazel] cache set to read-only"
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+ build --noremote_upload_local_results
+EOF
+fi
+
+if [[ "${BAZEL_CACHE_MODE:-none}" != @(read|read-write|none|) ]]; then
+ echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [read,read-write,none]"
+ exit 1
+fi

.buildkite/scripts/common/util.sh

@@ -18,7 +18,7 @@ verify_no_git_changes() {
  RED='\033[0;31m'
  C_RESET='\033[0m' # Reset color
 
- GIT_CHANGES="$(git ls-files --modified)"
+ GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
  if [ "$GIT_CHANGES" ]; then
  echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
  echo -e "$GIT_CHANGES\n"

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -73,6 +73,10 @@ const uploadPipeline = (pipelineContent) => {
  // pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
  // }
 
+ if (await doAnyChangesMatch([/^x-pack\/plugins\/uptime/])) {
+ pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime.yml'));
+ }
+
  pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
  uploadPipeline(pipeline.join('\n'));

.buildkite/scripts/steps/functional/common.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+# Note, changes here might also need to be made in other scripts, e.g. uptime.sh
+
 source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh