Merge branch 'main' into entity-risk-scoring-new-doc-link

.buildkite/ftr_configs.yml

@@ -124,6 +124,7 @@ enabled:
   - test/functional/apps/kibana_overview/config.ts
   - test/functional/apps/management/config.ts
   - test/functional/apps/saved_objects_management/config.ts
+  - test/functional/apps/sharing/config.ts
   - test/functional/apps/status_page/config.ts
   - test/functional/apps/visualize/group1/config.ts
   - test/functional/apps/visualize/group2/config.ts

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -147,6 +147,19 @@ steps:
             - exit_status: '*'
               limit: 1
 
+      - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+        label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 1
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
       - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
         label: 'Defend Workflows Cypress Tests on Serverless'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"

.buildkite/pipelines/flaky_tests/groups.json

@@ -49,7 +49,6 @@
       "key": "cypress/security_serverless_detection_engine",
       "name": "[Serverless] Security Solution Detection Engine - Cypress"
     },
-
     {
       "key": "cypress/security_solution_detection_engine_exceptions",
       "name": "Security Solution Detection Engine - Exceptions - Cypress"
@@ -58,6 +57,14 @@
       "key": "cypress/security_serverless_detection_engine_exceptions",
       "name": "[Serverless] Security Solution Detection Engine - Exceptions - Cypress"
     },
+    {
+      "key": "cypress/security_solution_ai_assistant",
+      "name": "Security Solution AI Assistant - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_ai_assistant",
+      "name": "[Serverless] Security Solution AI Assistant - Cypress"
+    },
     {
       "key": "cypress/defend_workflows",
       "name": "Security Solution Defend Workflows - Cypress"

.buildkite/pipelines/on_merge.yml

@@ -211,6 +211,30 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+    label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
+    label: 'AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/base.yml

@@ -141,6 +141,18 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
+    label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1        
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
@@ -213,6 +225,18 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
+    label: 'AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1        
+
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'
     agents:
@@ -231,7 +255,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 10
+    parallelism: 16
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -81,4 +81,16 @@ steps:
     retry:
       automatic:
         - exit_status: '*'
-          limit: 1
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:ai_assistant
+    label: 'Serverless MKI QA AI Assistant - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1          

.buildkite/pipelines/sonarqube.yml

@@ -0,0 +1,4 @@
+
+steps:
+  - label: Placeholder
+    command: echo "Hello!"

.buildkite/pull_requests.json

@@ -84,7 +84,8 @@
         "^\\.backportrc\\.json$",
         "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$"
+        "^\\.buildkite/pull_requests\\.json$",
+        "^\\.catalog-info\\.yaml$"
       ],
       "always_require_ci_on_changed": [
         "^docs/developer/plugin-list.asciidoc$",

.buildkite/scripts/common/util.sh

@@ -170,3 +170,34 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
+
+
+vault_get() {
+  path=$1
+  field=$2
+
+  fullPath="secret/ci/elastic-kibana/$path"
+  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+    fullPath="secret/kibana-issues/dev/$path"
+  fi
+
+  if [[ -z "${2:-}" ]]; then
+    retry 5 5 vault read "$fullPath"
+  else
+    retry 5 5 vault read -field="$field" "$fullPath"
+  fi
+}
+
+vault_set() {
+  path=$1
+  shift
+  fields=("$@")
+
+  fullPath="secret/ci/elastic-kibana/$path"
+  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+    fullPath="secret/kibana-issues/dev/$path"
+  fi
+
+  # shellcheck disable=SC2068
+  retry 5 5 vault write "$fullPath" ${fields[@]}
+}

.buildkite/scripts/lifecycle/pre_command.sh

@@ -8,7 +8,7 @@ echo '--- Setup environment vars'
 source .buildkite/scripts/common/env.sh
 source .buildkite/scripts/common/setup_node.sh
 
-BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
 export BUILDKITE_TOKEN
 
 echo '--- Install/build buildkite dependencies'
@@ -77,11 +77,11 @@ EOF
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
   export CI_STATS_BUILD_ID
-  
-  CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
+
+  CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
   export CI_STATS_TOKEN
-  
-  CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
+
+  CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
   export CI_STATS_HOST
 
   if [[ "$CI_STATS_BUILD_ID" ]]; then
@@ -97,66 +97,66 @@ EOF
   fi
 }
 
-GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
+GITHUB_TOKEN=$(vault_get kibanamachine github_token)
 export GITHUB_TOKEN
 
-KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
+KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
 export KIBANA_CI_GITHUB_TOKEN
 
-KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
+KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
 export KIBANA_CI_REPORTER_KEY
 
-KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
 export KIBANA_DOCKER_USERNAME
 
-KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
 export KIBANA_DOCKER_PASSWORD
 
-EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
+EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
 export EC_API_KEY
 
-PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
+PROJECT_API_KEY="$(vault_get kibana-ci-project-deploy pr_deploy_api_key)"
 export PROJECT_API_KEY
 
-PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
+PROJECT_API_DOMAIN="$(vault_get kibana-ci-project-deploy pr_deploy_domain)"
 export PROJECT_API_DOMAIN
 
-SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
 export SYNTHETICS_SERVICE_USERNAME
 
-SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
 export SYNTHETICS_SERVICE_PASSWORD
 
-SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
 export SYNTHETICS_SERVICE_MANIFEST
 
-SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
 export SYNTHETICS_REMOTE_KIBANA_USERNAME
 
-SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
 export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 
-SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
+SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
-DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
+DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
 export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
 
 # Setup Failed Test Reporter Elasticsearch credentials
 {
-  TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
   export TEST_FAILURES_ES_CLOUD_ID
 
-  TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
   export TEST_FAILURES_ES_USERNAME
 
-  TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
   export TEST_FAILURES_ES_PASSWORD
 }
 
 BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
 export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
-retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
 
 PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
 if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then

.buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh

@@ -14,7 +14,7 @@ echo "--- Serverless Security Second Quality Gate"
 cd x-pack/test/security_solution_api_integration
 set +e
 
-QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
 
 # Generate a random 5-digit number
 random_number=$((10000 + $RANDOM % 90000))
@@ -47,7 +47,7 @@ while : ; do
     echo "Sleeping for 40s to wait for ES status to be green..."
     sleep 40
   else
-    echo "Elasticsearch has status green." 
+    echo "Elasticsearch has status green."
     break
   fi
 done
@@ -59,17 +59,17 @@ while : ; do
     echo "Sleeping for 15s to wait for Kibana to be available..."
     sleep 15
   else
-    echo "Kibana is available." 
+    echo "Kibana is available."
     break
   fi
 done
 
 # Removing the https:// part of the url provided in order to use it in the command below.
-FORMATTED_ES_URL="${ES_URL/https:\/\//}"    
+FORMATTED_ES_URL="${ES_URL/https:\/\//}"
 FORMATTED_KB_URL="${KB_URL/https:\/\//}"
 
 # Find a way to remove this in the future
-# This is used in order to wait for the environment to be ready. 
+# This is used in order to wait for the environment to be ready.
 sleep 150
 
 TEST_CLOUD=1 TEST_ES_URL="https://elastic:$PASSWORD@$FORMATTED_ES_URL:443" TEST_KIBANA_URL="https://elastic:$PASSWORD@$FORMATTED_KB_URL:443" yarn run $1
@@ -79,4 +79,4 @@ echo "Exit code with status: $cmd_status"
 curl --location --request DELETE "https://global.qa.cld.elstc.co/api/v1/serverless/projects/security/$ID" \
   --header "Authorization: ApiKey $QA_API_KEY"
 
-exit $cmd_status
+exit $cmd_status

.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh

@@ -19,6 +19,6 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true
 cd x-pack/test/security_solution_cypress
 set +e
 
-QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
 
-CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status
+CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -86,7 +86,7 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
-  retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -121,14 +121,20 @@ fi
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
 
+if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+else
+  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+fi
+
 cat << EOF | buildkite-agent annotate --style "info" --context cloud
   ### Cloud Deployment
 
   Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL
 
   Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-  Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
+  Credentials: \`vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
 
   Kibana image: \`$KIBANA_CLOUD_IMAGE\`