Merge branch 'main' into fix_data_view_timestamp_validation

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.7",
     "8.6",
     "8.5",
     "8.4",
@@ -43,7 +44,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.7.0$": "main",
+    "^v8.8.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_configs.yml

@@ -4,11 +4,14 @@ disabled:
 
   # Base config files, only necessary to inform config finding script
   - test/functional/config.base.js
+  - test/functional/firefox/config.base.ts
   - x-pack/test/functional/config.base.js
   - x-pack/test/detection_engine_api_integration/security_and_spaces/config.base.ts
   - x-pack/test/functional_enterprise_search/base_config.ts
   - x-pack/test/localization/config.base.ts
   - test/server_integration/config.base.js
+  - x-pack/test/functional_with_es_ssl/config.base.ts
+  - x-pack/test/api_integration/config.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -23,6 +26,7 @@ disabled:
   - x-pack/test/functional_enterprise_search/cypress.config.ts
   - x-pack/test/defend_workflows_cypress/cli_config.ts
   - x-pack/test/defend_workflows_cypress/config.ts
+  - x-pack/test/defend_workflows_cypress/endpoint_config.ts
   - x-pack/test/defend_workflows_cypress/visual_config.ts
   - x-pack/test/osquery_cypress/cli_config.ts
   - x-pack/test/osquery_cypress/config.ts
@@ -43,12 +47,15 @@ disabled:
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/plugins/profiling/e2e/ftr_config.ts
 
   # Elastic Synthetics configs
   - x-pack/plugins/synthetics/e2e/config.ts
   - x-pack/plugins/synthetics/e2e/synthetics_run.ts
   - x-pack/plugins/ux/e2e/synthetics_run.ts
-  - x-pack/plugins/observability/e2e/synthetics_run.ts
+  - x-pack/plugins/exploratory_view/e2e/synthetics_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
@@ -64,6 +71,9 @@ disabled:
   # Scalability testing config that we run in its own pipeline
   - x-pack/test/scalability/config.ts
 
+  # Asset Manager configs, in tech preview, will move to enabled after more stability introduced
+  - x-pack/test/api_integration/apis/asset_manager/config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -84,6 +94,7 @@ enabled:
   - test/functional/apps/discover/embeddable/config.ts
   - test/functional/apps/discover/group1/config.ts
   - test/functional/apps/discover/group2/config.ts
+  - test/functional/apps/discover/group3/config.ts
   - test/functional/apps/getting_started/config.ts
   - test/functional/apps/home/config.ts
   - test/functional/apps/kibana_overview/config.ts
@@ -98,7 +109,11 @@ enabled:
   - test/functional/apps/visualize/group6/config.ts
   - test/functional/apps/visualize/replaced_vislib_chart_types/config.ts
   - test/functional/config.ccs.ts
-  - test/functional/config.firefox.js
+  - test/functional/firefox/console.config.ts
+  - test/functional/firefox/dashboard.config.ts
+  - test/functional/firefox/discover.config.ts
+  - test/functional/firefox/home.config.ts
+  - test/functional/firefox/visualize.config.ts
   - test/health_gateway/config.ts
   - test/interactive_setup_api_integration/enrollment_flow.config.ts
   - test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
@@ -137,7 +152,37 @@ enabled:
   - x-pack/test/api_integration_basic/config.ts
   - x-pack/test/api_integration/config_security_basic.ts
   - x-pack/test/api_integration/config_security_trial.ts
-  - x-pack/test/api_integration/config.ts
+  - x-pack/test/api_integration/apis/aiops/config.ts
+  - x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
+  - x-pack/test/api_integration/apis/cases/config.ts
+  - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
+  - x-pack/test/api_integration/apis/console/config.ts
+  - x-pack/test/api_integration/apis/es/config.ts
+  - x-pack/test/api_integration/apis/features/config.ts
+  - x-pack/test/api_integration/apis/file_upload/config.ts
+  - x-pack/test/api_integration/apis/kibana/config.ts
+  - x-pack/test/api_integration/apis/lists/config.ts
+  - x-pack/test/api_integration/apis/logs_ui/config.ts
+  - x-pack/test/api_integration/apis/logstash/config.ts
+  - x-pack/test/api_integration/apis/management/config.ts
+  - x-pack/test/api_integration/apis/maps/config.ts
+  - x-pack/test/api_integration/apis/metrics_ui/config.ts
+  - x-pack/test/api_integration/apis/ml/config.ts
+  - x-pack/test/api_integration/apis/monitoring/config.ts
+  - x-pack/test/api_integration/apis/monitoring_collection/config.ts
+  - x-pack/test/api_integration/apis/osquery/config.ts
+  - x-pack/test/api_integration/apis/painless_lab/config.ts
+  - x-pack/test/api_integration/apis/search/config.ts
+  - x-pack/test/api_integration/apis/searchprofiler/config.ts
+  - x-pack/test/api_integration/apis/security/config.ts
+  - x-pack/test/api_integration/apis/security_solution/config.ts
+  - x-pack/test/api_integration/apis/spaces/config.ts
+  - x-pack/test/api_integration/apis/synthetics/config.ts
+  - x-pack/test/api_integration/apis/telemetry/config.ts
+  - x-pack/test/api_integration/apis/transform/config.ts
+  - x-pack/test/api_integration/apis/upgrade_assistant/config.ts
+  - x-pack/test/api_integration/apis/uptime/config.ts
+  - x-pack/test/api_integration/apis/watcher/config.ts
   - x-pack/test/apm_api_integration/basic/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
@@ -180,7 +225,10 @@ enabled:
   - x-pack/test/functional_embedded/config.ts
   - x-pack/test/functional_enterprise_search/without_host_configured.config.ts
   - x-pack/test/functional_execution_context/config.ts
-  - x-pack/test/functional_with_es_ssl/config.ts
+  - x-pack/test/functional_with_es_ssl/apps/cases/group1/config.ts
+  - x-pack/test/functional_with_es_ssl/apps/cases/group2/config.ts
+  - x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/config.ts
+  - x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/config.ts
   - x-pack/test/functional/apps/advanced_settings/config.ts
   - x-pack/test/functional/apps/aiops/config.ts
   - x-pack/test/functional/apps/api_keys/config.ts
@@ -243,6 +291,7 @@ enabled:
   - x-pack/test/functional/config_security_basic.ts
   - x-pack/test/functional/config.ccs.ts
   - x-pack/test/functional/config.firefox.js
+  - x-pack/test/functional_cloud/config.ts
   - x-pack/test/kubernetes_security/basic/config.ts
   - x-pack/test/licensing_plugin/config.public.ts
   - x-pack/test/licensing_plugin/config.ts
@@ -318,3 +367,5 @@ enabled:
   - x-pack/performance/journeys/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys/dashboard_listing_page.ts
+  - x-pack/performance/journeys/cloud_security_dashboard.ts
+  - x-pack/test/custom_branding/config.ts

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -179,6 +179,15 @@ export async function pickTestGroupRunOrder() {
     throw new Error(`invalid FUNCTIONAL_MAX_MINUTES: ${process.env.FUNCTIONAL_MAX_MINUTES}`);
   }
 
+  /**
+   * This env variable corresponds to the env stanza within
+   * https://github.com/elastic/kibana/blob/bc2cb5dc613c3d455a5fed9c54450fd7e46ffd92/.buildkite/pipelines/code_coverage/daily.yml#L17
+   *
+   * It is a flag that signals the job for which test runners will be executed.
+   *
+   * For example in code coverage pipeline definition, it is "limited"
+   * to 'unit,integration'.  This means FTR tests will not be executed.
+   */
   const LIMIT_CONFIG_TYPE = process.env.LIMIT_CONFIG_TYPE
     ? process.env.LIMIT_CONFIG_TYPE.split(',')
         .map((t) => t.trim())
@@ -218,9 +227,10 @@ export async function pickTestGroupRunOrder() {
       : ['build'];
 
   const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
-  if (!LIMIT_CONFIG_TYPE.includes('functional')) {
-    ftrConfigsByQueue.clear();
-  }
+
+  const ftrConfigsIncluded = LIMIT_CONFIG_TYPE.includes('functional');
+
+  if (!ftrConfigsIncluded) ftrConfigsByQueue.clear();
 
   const jestUnitConfigs = LIMIT_CONFIG_TYPE.includes('unit')
     ? globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], {
@@ -377,9 +387,11 @@ export async function pickTestGroupRunOrder() {
   Fs.writeFileSync('jest_run_order.json', JSON.stringify({ unit, integration }, null, 2));
   bk.uploadArtifacts('jest_run_order.json');
 
-  // write the config for functional steps to an artifact that can be used by the individual functional jobs
-  Fs.writeFileSync('ftr_run_order.json', JSON.stringify(ftrRunOrder, null, 2));
-  bk.uploadArtifacts('ftr_run_order.json');
+  if (ftrConfigsIncluded) {
+    // write the config for functional steps to an artifact that can be used by the individual functional jobs
+    Fs.writeFileSync('ftr_run_order.json', JSON.stringify(ftrRunOrder, null, 2));
+    bk.uploadArtifacts('ftr_run_order.json');
+  }
 
   // upload the step definitions to Buildkite
   bk.uploadSteps(

.buildkite/pipelines/artifacts_trigger.yml

@@ -0,0 +1,6 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/trigger.sh
+    label: Trigger artifacts build
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10

.buildkite/pipelines/code_coverage/daily.yml

@@ -13,18 +13,16 @@ steps:
       queue: kibana-default
     env:
       FTR_CONFIGS_DEPS: ''
-      #      LIMIT_CONFIG_TYPE: 'unit,functional,integration'
       LIMIT_CONFIG_TYPE: 'unit,integration'
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest.sh'
       JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest_integration.sh'
 
   - command: .buildkite/scripts/steps/code_coverage/ingest.sh
     label: 'Merge and Ingest'
     agents:
-      queue: c2-16
+      queue: n2-4-spot
     depends_on:
       - jest
       - jest-integration
-    #      - ftr-configs
     timeout_in_minutes: 30
     key: ingest

.buildkite/pipelines/on_merge.yml

@@ -78,6 +78,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
+    label: Trigger unsupported ftr tests
+    timeout_in_minutes: 10
+    depends_on:
+      - build
+    agents:
+      queue: 'kibana-default'
+
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -0,0 +1,79 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: n2-16-spot
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution.sh
+    label: 'Security Solution Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
+    label: 'Synthetics @elastic/synthetics Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    artifact_paths:
+      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/check_next_docs.yml

@@ -0,0 +1,10 @@
+steps:
+  - command: .buildkite/scripts/steps/next_docs/build_and_validate_docs.sh
+    label: 'Build and Validate Next Docs'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/exploratory_view_plugin.yml

@@ -0,0 +1,15 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/observability_plugin.sh
+    label: 'Exploratory View @elastic/synthetics Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    artifact_paths:
+      - 'x-pack/plugins/exploratory_view/e2e/.journeys/**/*'
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/profiling_cypress.yml

@@ -0,0 +1,14 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profiling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/threat_intelligence.yml

@@ -12,3 +12,5 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+    artifact_paths:
+      - "target/kibana-threat-intelligence/**/*"