Merge branch 'main' into unified-search-1

.buildkite/ftr_configs.yml

@@ -34,7 +34,10 @@ disabled:
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+
+  # Elastic Synthetics configs
   - x-pack/plugins/synthetics/e2e/config.ts
+  - x-pack/plugins/synthetics/e2e/playwright_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
   - test/visual_regression/config.ts

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#ef419d4f761dd256cb59bfab9b59f8b91029eb40"
   }
 }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -32,6 +32,9 @@ steps:
     agents:
       queue: kibana-default
     env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
       LIMIT_CONFIG_TYPE: integration,functional
     retry:
       automatic:
@@ -44,10 +47,8 @@ steps:
     agents:
       queue: kibana-default
     depends_on:
-      - default-cigroup
-      - oss-cigroup
+      - ftr-configs
       - jest-integration
-      - api-integration
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/on_merge.yml

@@ -53,6 +53,10 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       queue: kibana-default
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
@@ -78,10 +82,19 @@ steps:
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
     agents:
       queue: c2-8
-    key: checks
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/pipelines/performance/daily.yml

@@ -17,6 +17,13 @@ steps:
     agents:
       queue: kb-static-ubuntu
     depends_on: build
+    key: tests
+
+  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/pull_request/base.yml

@@ -19,6 +19,10 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       queue: kibana-default
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
@@ -40,10 +44,19 @@ steps:
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
     agents:
       queue: c2-8
-    key: checks
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/scripts/bootstrap.sh

@@ -6,6 +6,15 @@ source .buildkite/scripts/common/util.sh
 source .buildkite/scripts/common/setup_bazel.sh
 
 echo "--- yarn install and bootstrap"
+
+# Use the node_modules that is baked into the agent image, if it exists, as a cache
+# But only for agents not mounting the workspace on a local ssd or in memory
+# It actually ends up being slower to move all of the tiny files between the disks vs extracting archives from the yarn cache
+if [[ -d ~/.kibana/node_modules && "$(pwd)" != *"/local-ssd/"* && "$(pwd)" != "/dev/shm"* ]]; then
+  echo "Using ~/.kibana/node_modules as a starting point"
+  mv ~/.kibana/node_modules ./
+fi
+
 if ! yarn kbn bootstrap; then
   echo "bootstrap failed, trying again in 15 seconds"
   sleep 15

.buildkite/scripts/common/env.sh

@@ -101,3 +101,8 @@ export DISABLE_BOOTSTRAP_VALIDATION=true
 
 # Prevent Browserlist from logging on CI about outdated database versions
 export BROWSERSLIST_IGNORE_OLD_DATA=true
+
+# keys used to associate test group data in ci-stats with Jest execution order
+export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
+export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"
+export TEST_GROUP_TYPE_FUNCTIONAL="Functional Tests"

.buildkite/scripts/steps/checks/check_types.sh → .buildkite/scripts/steps/check_types.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
+.buildkite/scripts/bootstrap.sh
+
 echo --- Check Types
 checks-reporter-with-killswitch "Check Types" \
   node scripts/type_check

.buildkite/scripts/steps/checks.sh

@@ -13,7 +13,6 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/doc_api_changes.sh
 .buildkite/scripts/steps/checks/kbn_pm_dist.sh
 .buildkite/scripts/steps/checks/plugin_list_docs.sh
-.buildkite/scripts/steps/checks/check_types.sh
 .buildkite/scripts/steps/checks/bundle_limits.sh
 .buildkite/scripts/steps/checks/i18n.sh
 .buildkite/scripts/steps/checks/file_casing.sh

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
+PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
+ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
+BUILD_ID=${BUILDKITE_BUILD_ID}
+GCS_BUCKET="gs://kibana-performance/scalability-tests"
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Extract APM metrics"
+journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
+
+for i in "${journeys[@]}"; do
+    JOURNEY_NAME="${i}"
+    echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"
+
+    node scripts/extract_performance_testing_dataset \
+        --journeyName "${JOURNEY_NAME}" \
+        --buildId "${BUILD_ID}" \
+        --es-url "${ES_SERVER_URL}" \
+        --es-username "${USER_FROM_VAULT}" \
+        --es-password "${PASS_FROM_VAULT}"
+done
+
+echo "--- Upload Kibana build, plugins and scalability traces to the public bucket"
+mkdir "${BUILD_ID}"
+# Archive json files with traces and upload as build artifacts
+tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
+buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
+# Upload Kibana build, plugins, commit sha and traces to the bucket
+buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
+buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
+echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
+gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
+echo "--- Update reference to the latest CI build"
+echo "${BUILD_ID}" > LATEST
+gsutil cp LATEST "${GCS_BUCKET}"

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -7,6 +7,4 @@ set -euo pipefail
 .buildkite/scripts/build_kibana_plugins.sh
 .buildkite/scripts/post_build_kibana_plugins.sh
 .buildkite/scripts/post_build_kibana.sh
-
-source ".buildkite/scripts/steps/test/test_group_env.sh"
 .buildkite/scripts/saved_object_field_metrics.sh

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/test/test_group_env.sh
 
 export JOB_NUM=$BUILDKITE_PARALLEL_JOB
 export JOB=ftr-configs-${JOB_NUM}

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/steps/test/test_group_env.sh
-
 export JOB=$BUILDKITE_PARALLEL_JOB
 
 # a jest failure will result in the script returning an exit code of 10

.buildkite/scripts/steps/test/pick_test_group_run_order.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/test/test_group_env.sh
 
 echo '--- Pick Test Group Run Order'
 node "$(dirname "${0}")/pick_test_group_run_order.js"

.github/CODEOWNERS

@@ -128,6 +128,7 @@
 /src/apm.js @elastic/kibana-core @vigneshshanmugam
 /packages/kbn-apm-config-loader/ @elastic/kibana-core @vigneshshanmugam
 /src/core/types/elasticsearch @elastic/apm-ui
+/packages/elastic-apm-synthtrace/  @elastic/apm-ui
 #CC# /src/plugins/apm_oss/ @elastic/apm-ui
 #CC# /x-pack/plugins/observability/ @elastic/apm-ui
 

package.json

@@ -252,7 +252,7 @@
     "deep-freeze-strict": "^1.1.1",
     "deepmerge": "^4.2.2",
     "del": "^5.1.0",
-    "elastic-apm-node": "^3.32.0",
+    "elastic-apm-node": "^3.33.0",
     "email-addresses": "^5.0.0",
     "execa": "^4.0.2",
     "exit-hook": "^2.2.0",
@@ -500,6 +500,7 @@
     "@kbn/import-resolver": "link:bazel-bin/packages/kbn-import-resolver",
     "@kbn/jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers",
     "@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer",
+    "@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor",
     "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator",
     "@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers",
     "@kbn/pm": "link:packages/kbn-pm",
@@ -646,6 +647,7 @@
     "@types/kbn__mapbox-gl": "link:bazel-bin/packages/kbn-mapbox-gl/npm_module_types",
     "@types/kbn__monaco": "link:bazel-bin/packages/kbn-monaco/npm_module_types",
     "@types/kbn__optimizer": "link:bazel-bin/packages/kbn-optimizer/npm_module_types",
+    "@types/kbn__performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor/npm_module_types",
     "@types/kbn__plugin-discovery": "link:bazel-bin/packages/kbn-plugin-discovery/npm_module_types",
     "@types/kbn__plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator/npm_module_types",
     "@types/kbn__plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers/npm_module_types",

packages/BUILD.bazel

@@ -62,6 +62,7 @@ filegroup(
     "//packages/kbn-mapbox-gl:build",
     "//packages/kbn-monaco:build",
     "//packages/kbn-optimizer:build",
+    "//packages/kbn-performance-testing-dataset-extractor:build",
     "//packages/kbn-plugin-discovery:build",
     "//packages/kbn-plugin-generator:build",
     "//packages/kbn-plugin-helpers:build",
@@ -160,6 +161,7 @@ filegroup(
     "//packages/kbn-mapbox-gl:build_types",
     "//packages/kbn-monaco:build_types",
     "//packages/kbn-optimizer:build_types",
+    "//packages/kbn-performance-testing-dataset-extractor:build_types",
     "//packages/kbn-plugin-discovery:build_types",
     "//packages/kbn-plugin-generator:build_types",
     "//packages/kbn-plugin-helpers:build_types",

packages/analytics/client/BUILD.bazel

@@ -62,6 +62,13 @@ jsts_transpiler(
   build_pkg_name = package_name(),
 )
 
+jsts_transpiler(
+  name = "target_web",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+  web = True,
+)
+
 ts_config(
   name = "tsconfig",
   src = "tsconfig.json",
@@ -86,7 +93,7 @@ ts_project(
 js_library(
   name = PKG_DIRNAME,
   srcs = NPM_MODULE_EXTRA_FILES,
-  deps = RUNTIME_DEPS + [":target_node"],
+  deps = RUNTIME_DEPS + [":target_node", ":target_web"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],
 )

packages/analytics/client/package.json

@@ -2,6 +2,7 @@
   "name": "@kbn/analytics-client",
   "private": true,
   "version": "1.0.0",
+  "browser": "./target_web/index.js",
   "main": "./target_node/index.js",
   "license": "SSPL-1.0 OR Elastic License 2.0"
 }

packages/analytics/shippers/fullstory/BUILD.bazel

@@ -62,6 +62,13 @@ jsts_transpiler(
   build_pkg_name = package_name(),
 )
 
+jsts_transpiler(
+  name = "target_web",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+  web = True,
+)
+
 ts_config(
   name = "tsconfig",
   src = "tsconfig.json",
@@ -86,7 +93,7 @@ ts_project(
 js_library(
   name = PKG_DIRNAME,
   srcs = NPM_MODULE_EXTRA_FILES,
-  deps = RUNTIME_DEPS + [":target_node"],
+  deps = RUNTIME_DEPS + [":target_node", ":target_web"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],
 )

packages/analytics/shippers/fullstory/package.json

@@ -2,6 +2,7 @@
   "name": "@kbn/analytics-shippers-fullstory",
   "private": true,
   "version": "1.0.0",
+  "browser": "./target_web/index.js",
   "main": "./target_node/index.js",
   "license": "SSPL-1.0 OR Elastic License 2.0"
 }