Merge branch 'main' of github.com:elastic/kibana into mock-synthetic-…

…source

.buildkite/ftr_configs.yml

@@ -117,6 +117,7 @@ enabled:
   - x-pack/test/alerting_api_integration/basic/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/spaces_only/config.ts
   - x-pack/test/api_integration_basic/config.ts
   - x-pack/test/api_integration/config_security_basic.ts
@@ -250,6 +251,7 @@ enabled:
   - x-pack/test/security_functional/oidc.config.ts
   - x-pack/test/security_functional/saml.config.ts
   - x-pack/test/security_functional/insecure_cluster_warning.config.ts
+  - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
   - x-pack/test/security_solution_endpoint/config.ts
   - x-pack/test/session_view/basic/config.ts

.buildkite/pipeline-utils/test-failures/annotate.ts

@@ -150,7 +150,7 @@ export const annotateTestFailures = async () => {
   }
 
   exec(
-    `buildkite-agent artifact download --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
+    `.buildkite/scripts/common/download_artifact.sh --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
   );
 
   const failures: TestFailure[] = recursiveReadDir(failureDir)

.buildkite/pipelines/on_merge.yml

@@ -135,7 +135,7 @@ steps:
     label: 'Populate local dev bazel cache (Linux)'
     agents:
       queue: n2-4-spot
-    timeout_in_minutes: 15
+    timeout_in_minutes: 30
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/performance/daily.yml

@@ -11,13 +11,15 @@ steps:
     agents:
       queue: c2-16
     key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
   - label: ':muscle: Performance Tests with Playwright config'
     command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
       queue: kb-static-ubuntu
     depends_on: build
     key: tests
+    timeout_in_minutes: 60
 
   - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
     command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

.buildkite/scripts/common/download_artifact.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "$0")/util.sh"
+download_artifact "$@"

.buildkite/scripts/common/env.sh

@@ -26,10 +26,22 @@ export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_VERSION
 
-export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
+BUILDKITE_AGENT_GCP_REGION=""
+if [[ "$(curl -is metadata.google.internal || true)" ]]; then
+  # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
+  BUILDKITE_AGENT_GCP_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
+fi
+export BUILDKITE_AGENT_GCP_REGION
+
+CI_PROXY_CACHE_SUFFIX=""
+if [[ "$BUILDKITE_AGENT_GCP_REGION" ]]; then
+  CI_PROXY_CACHE_SUFFIX="/region/$BUILDKITE_AGENT_GCP_REGION"
+fi
+
+export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX/cypress"
 
 export NODE_OPTIONS="--max-old-space-size=4096"
 

.buildkite/scripts/common/setup_bazel.sh

@@ -16,12 +16,7 @@ BAZEL_CACHE_MODE=${BAZEL_CACHE_MODE:-gcs}
 if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
   echo "[bazel] enabling caching with GCS buckets"
 
-  BAZEL_REGION="us-central1"
-  if [[ "$(curl -is metadata.google.internal || true)" ]]; then
-    # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
-    BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
-  fi
-
+  BAZEL_REGION="${BUILDKITE_AGENT_GCP_REGION:-us-central1}"
   BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION"
 
   echo "[bazel] using GCS bucket: $BAZEL_BUCKET"

.buildkite/scripts/common/util.sh

@@ -168,3 +168,9 @@ npm_install_global() {
     npm install -g "$toInstall" && touch "$successFlag"
   fi
 }
+
+# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage
+# times-out after 60 seconds and retries up to 3 times
+download_artifact() {
+  retry 3 1 timeout 3m buildkite-agent artifact download "$@"
+}

.buildkite/scripts/download_build_artifacts.sh

@@ -2,13 +2,15 @@
 
 set -euo pipefail
 
+source "$(dirname "$0")/common/util.sh"
+
 if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
   echo '--- Downloading Distribution and Plugin artifacts'
 
   cd "$WORKSPACE"
 
-  buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
   mkdir -p "$KIBANA_BUILD_LOCATION"
   tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -122,7 +122,11 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:deploy-cloud')) {
+    if (
+      GITHUB_PR_LABELS.includes('ci:deploy-cloud') ||
+      GITHUB_PR_LABELS.includes('ci:cloud-deploy') ||
+      GITHUB_PR_LABELS.includes('ci:cloud-redeploy')
+    ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -4,12 +4,13 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Build and publish Cloud image"
 mkdir -p target
 
-buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 node scripts/build \
   --skip-initialize \

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
 KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
@@ -28,7 +29,7 @@ fi
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
-buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/artifacts/publish.sh

@@ -7,8 +7,8 @@ source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Download and verify artifacts"
 function download {
-  buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   sha512sum -c "$1.sha512.txt"
   rm "$1.sha512.txt"
 }
@@ -40,7 +40,7 @@ download "kibana-$FULL_VERSION-windows-x86_64.zip"
 
 download "dependencies-$FULL_VERSION.csv"
 
-cd - 
+cd -
 
 echo "--- Set artifact permissions"
 chmod -R a+r target/*

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -9,11 +9,12 @@ source .buildkite/scripts/common/util.sh
 export KBN_NP_PLUGINS_BUILT=true
 
 VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
+ECCTL_LOGS=$(mktemp --suffix ".json")
 
 echo "--- Download Kibana Distribution"
 
 mkdir -p ./target
-buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build Cloud Distribution"
 ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
@@ -46,9 +47,19 @@ fi
 
 docker logout docker.elastic.co
 
+if is_pr_with_label "ci:cloud-redeploy"; then
+  echo "--- Shutdown Previous Deployment"
+  CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
+  if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+    echo "No deployment to remove"
+  else
+    echo "Shutting down previous deployment..."
+    ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json > "$ECCTL_LOGS"
+  fi
+fi
+
 echo "--- Create Deployment"
 CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
@@ -60,45 +71,49 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
     .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'"
     ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
 
-  echo -n "Creating deployment..."
-  ecctl deployment create --track --output json --file /tmp/deploy.json > "$JSON_FILE"
-  echo "done"
+  echo "Creating deployment..."
+  ecctl deployment create --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 
-  CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$JSON_FILE")
+  CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$ECCTL_LOGS")
 
-  echo -n "Writing to vault..."
+  echo "Writing to vault..."
   VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
   retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
 
-  echo -n "Enabling Stack Monitoring..."
+  echo "Enabling Stack Monitoring..."
   jq '
     .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" |
     .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'"
     ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$JSON_FILE"
-  echo "done"
 
-  echo -n "Enabling verbose logging..."
+  # After a deployment is created, a new Kibana plan is automatically added to update settings
+  # and restart Kibana.  Polling for a plan state isn't especially reliable because it flips
+  # between empty and queued, and then empty again depending on how quickly setup is run in cloud.
+  # We want to enable monitoring after the automatic plan has run, if not we get an error:
+  # * deployments.resource_plan_state_error: Kibana resource [main-kibana] has a plan still pending, cancel that or wait for it to complete (settings.observability.plan)
+  # This adds a sleep and retry to see if we can make this step more reliable
+  sleep 120
+  retry 5 60 ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$ECCTL_LOGS"
+
+  echo "Enabling verbose logging..."
   ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
     .resources.kibana[0].plan.kibana.user_settings_yaml = "logging.root.level: all"
     ' > /tmp/verbose_logging.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/verbose_logging.json > "$JSON_FILE"
-  echo "done"
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/verbose_logging.json > "$ECCTL_LOGS"
 else
-ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
-  .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
-  (.. | select(.version? != null).version) = "'$VERSION'"
-  ' > /tmp/deploy.json
-
-  echo -n "Updating deployment..."
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$JSON_FILE"
-  echo "done"
+  ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
+    .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
+    (.. | select(.version? != null).version) = "'$VERSION'"
+    ' > /tmp/deploy.json
+
+  echo "Updating deployment..."
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
@@ -119,3 +134,4 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 EOF
 
 buildkite-agent meta-data set pr_comment:deploy_cloud:head "* [Cloud Deployment](${CLOUD_DEPLOYMENT_KIBANA_URL})"
+buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -31,9 +31,13 @@ for (const deployment of prDeployments) {
     if (pullRequest.state !== 'OPEN') {
       console.log(`Pull Request #${prNumber} is no longer open, will delete associated deployment`);
       deploymentsToPurge.push(deployment);
-    } else if (!pullRequest.labels.filter((label: any) => label.name === 'ci:deploy-cloud')) {
+    } else if (
+      !pullRequest.labels.filter((label: any) =>
+        /^ci:(deploy-cloud|cloud-deploy|cloud-redeploy)$/.test(label.name)
+      )
+    ) {
       console.log(
-        `Pull Request #${prNumber} no longer has the ci:deploy-cloud label, will delete associated deployment`
+        `Pull Request #${prNumber} no longer has the a cloud deployment label, will delete associated deployment`
       );
       deploymentsToPurge.push(deployment);
     } else if (lastCommitTimestamp < NOW - 60 * 60 * 24 * 7) {