Merge branch 'main' into hosts-alerts-serverless

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.x",
     "8.15",
     "8.14",
     "8.13",
@@ -52,7 +53,8 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.16.0$": "main",
+    "^v9.0.0$": "main",
+    "^v8.16.0$": "8.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_security_stateful_configs.yml

@@ -21,8 +21,6 @@ disabled:
   - x-pack/test/osquery_cypress/visual_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
   - x-pack/test/security_solution_cypress/config.ts
-  - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
-  - x-pack/test/threat_intelligence_cypress/config.ts
 
   # Playwright
   - x-pack/test/security_solution_playwright/playwright.config.ts

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -49,6 +49,10 @@ spec:
           cronline: 0 9 * * * America/New_York
           message: Daily build
           branch: main
+        Daily build (8.x):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: '8.x'
         Daily build (8.15):
           cronline: 0 9 * * * America/New_York
           message: Daily build
@@ -82,7 +86,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -130,7 +134,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -34,5 +34,7 @@ spec:
       teams:
         kibana-operations:
           access_level: MANAGE_BUILD_AND_READ
+        kibana-security:
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.15 7.17
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 7.17 8.15
+      branch_configuration: main 8.x 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-utils/utils.test.ts

@@ -36,9 +36,6 @@ describe('getVersionsFile', () => {
     const versionsFile = getVersionsFile();
 
     expect(versionsFile.prevMajors).to.be.an('array');
-    expect(versionsFile.prevMajors.length).to.eql(1);
-    expect(versionsFile.prevMajors[0].branch).to.eql('7.17');
-
     expect(versionsFile.prevMinors).to.be.an('array');
   });
 

.buildkite/pipeline-utils/utils.ts

@@ -43,6 +43,7 @@ const getVersionsFile = (() => {
     prevMinors: Version[];
     prevMajors: Version[];
     current: Version;
+    versions: Version[];
   };
   const versionsFileName = 'versions.json';
   try {

.buildkite/pipelines/on_merge.yml

@@ -386,24 +386,6 @@ steps:
         - exit_status: '-1'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
-    label: 'Threat Intelligence Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on:
-      - build
-      - quick_checks
-    timeout_in_minutes: 60
-    parallelism: 1
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:

.buildkite/pipelines/pointer_compression.yml

@@ -321,23 +321,6 @@ steps:
         - exit_status: '-1'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
-    label: 'Threat Intelligence Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on:
-      - build
-    timeout_in_minutes: 60
-    parallelism: 1
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:

.buildkite/scripts/lifecycle/post_command.sh

@@ -35,6 +35,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
   buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
+  buildkite-agent artifact upload '.es/es*.log'
 
   if [[ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]]; then
     if [[ $BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG == 'elasticsearch-serverless-intake' ]]; then

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -245,22 +245,35 @@ const getPipeline = (filename: string, removeSteps = true) => {
     if (
       (await doAnyChangesMatch([
         /^package.json/,
+        /^packages\/kbn-doc-links/,
+        /^packages\/kbn-es-query/,
+        /^packages\/kbn-i18n-react/,
+        /^packages\/kbn-i18n/,
+        /^packages\/kbn-expandable-flyout/,
         /^packages\/kbn-securitysolution-.*/,
+        /^packages\/kbn-securitysolution-io-ts-list-types/,
+        /^packages\/shared-ux/,
+        /^src\/core/,
+        /^src\/plugins\/data/,
+        /^src\/plugins\/kibana_utils/,
+        /^src\/plugins\/inspector/,
+        /^x-pack\/packages\/kbn-elastic-assistant/,
+        /^x-pack\/packages\/kbn-elastic-assistant-common/,
+        /^x-pack\/packages\/security-solution/,
         /^x-pack\/plugins\/alerting/,
+        /^x-pack\/plugins\/cases/,
         /^x-pack\/plugins\/data_views\/common/,
+        /^x-pack\/plugins\/elastic_assistant/,
         /^x-pack\/plugins\/lists/,
         /^x-pack\/plugins\/rule_registry\/common/,
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/security_solution_ess/,
         /^x-pack\/plugins\/security_solution_serverless/,
         /^x-pack\/plugins\/task_manager/,
+        /^x-pack\/plugins\/threat_intelligence/,
         /^x-pack\/plugins\/timelines/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/alerts_table/,
         /^x-pack\/plugins\/usage_collection\/public/,
-        /^x-pack\/plugins\/elastic_assistant/,
-        /^x-pack\/packages\/security-solution/,
-        /^x-pack\/packages\/kbn-elastic-assistant/,
-        /^x-pack\/packages\/kbn-elastic-assistant-common/,
         /^x-pack\/test\/functional\/es_archives\/security_solution/,
         /^x-pack\/test\/security_solution_cypress/,
       ])) ||
@@ -271,37 +284,6 @@ const getPipeline = (filename: string, removeSteps = true) => {
       );
     }
 
-    if (
-      (await doAnyChangesMatch([
-        /^package.json/,
-        /^src\/plugins\/data/,
-        /^src\/plugins\/kibana_utils/,
-        /^src\/plugins\/inspector/,
-        /^src\/plugins\/data_views/,
-        /^src\/core/,
-        /^packages\/kbn-securitysolution-.*/,
-        /^packages\/kbn-es-query/,
-        /^packages\/kbn-securitysolution-io-ts-list-types/,
-        /^packages\/kbn-i18n-react/,
-        /^packages\/kbn-i18n/,
-        /^packages\/shared-ux/,
-        /^packages\/kbn-doc-links/,
-        /^packages\/kbn-securitysolution-io-ts-list-types/,
-        /^x-pack\/plugins\/threat_intelligence/,
-        /^x-pack\/packages\/security-solution/,
-        /^x-pack\/test\/threat_intelligence_cypress/,
-        /^x-pack\/plugins\/cases/,
-        /^x-pack\/plugins\/timelines/,
-        /^x-pack\/plugins\/triggers_actions_ui/,
-        /^x-pack\/plugins\/rule_registry/,
-      ])) ||
-      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
-    ) {
-      pipeline.push(
-        getPipeline('.buildkite/pipelines/pull_request/security_solution/threat_intelligence.yml')
-      );
-    }
-
     if (
       ((await doAnyChangesMatch([
         /^x-pack\/plugins\/osquery/,

.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts

@@ -25,7 +25,7 @@ describe('pipeline trigger combinations', () => {
   it('should trigger the correct pipelines for "es-forward"', () => {
     const esForwardTriggers = getESForwardPipelineTriggers();
     // tests 7.17 against 8.x versions
-    const targets = versionsFile.versions.filter((v) => v.currentMajor === true);
+    const targets = versionsFile.versions.filter((v) => v.branch.startsWith('8.'));
 
     expect(esForwardTriggers.length).to.eql(targets.length);
 
@@ -43,7 +43,6 @@ describe('pipeline trigger combinations', () => {
     const snapshotTriggers = getArtifactSnapshotPipelineTriggers();
     // triggers for all open branches
     const branches = versionsFile.versions.map((v) => v.branch);
-
     expect(snapshotTriggers.length).to.eql(branches.length);
 
     branches.forEach((b) => {
@@ -53,9 +52,10 @@ describe('pipeline trigger combinations', () => {
 
   it('should trigger the correct pipelines for "artifacts-trigger"', () => {
     const triggerTriggers = getArtifactBuildTriggers();
-    // all currentMajor+prevMinor branches
+    // all branches that have fixed versions, and excluding 7.17 (i.e. not 7.17, [0-9].x and main)
     const branches = versionsFile.versions
-      .filter((v) => v.currentMajor === true && v.previousMinor === true)
+      .filter((v) => v.branch.match(/[0-9]{1,2}\.[0-9]{1,2}/))
+      .filter((v) => v.previousMajor === true)
       .map((v) => v.branch);
 
     expect(triggerTriggers.length).to.eql(branches.length);
@@ -66,9 +66,9 @@ describe('pipeline trigger combinations', () => {
 
   it('should trigger the correct pipelines for "artifacts-staging"', () => {
     const stagingTriggers = getArtifactStagingPipelineTriggers();
-    // all branches that are not currentMajor+currentMinor
+    // all branches that have fixed versions (i.e. not [0-9].x and main)
     const branches = versionsFile.versions
-      .filter((v) => !v.currentMajor || !v.currentMinor)
+      .filter((v) => v.branch.match(/[0-9]{1,2}\.[0-9]{1,2}/))
       .map((v) => v.branch);
 
     expect(stagingTriggers.length).to.eql(branches.length);

.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts

@@ -66,17 +66,20 @@ async function main() {
  */
 export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] {
   const versions = getVersionsFile();
-  const kibanaPrevMajor = versions.prevMajors[0];
-  const targetESVersions = [versions.prevMinors, versions.current].flat();
+  const KIBANA_7_17 = versions.versions.find((v) => v.branch === '7.17');
+  if (!KIBANA_7_17) {
+    throw new Error('Update ES forward compatibility pipeline to remove 7.17 and add version 8');
+  }
+  const targetESVersions = versions.versions.filter((v) => v.branch.startsWith('8.'));
 
   return targetESVersions.map(({ version }) => {
     return {
       trigger: pipelineSets['es-forward'],
       async: true,
-      label: `Triggering Kibana ${kibanaPrevMajor.version} + ES ${version} forward compatibility`,
+      label: `Triggering Kibana ${KIBANA_7_17.version} + ES ${version} forward compatibility`,
       build: {
         message: process.env.MESSAGE || `ES forward-compatibility test for ES ${version}`,
-        branch: kibanaPrevMajor.branch,
+        branch: KIBANA_7_17.branch,
         commit: 'HEAD',
         env: {
           ES_SNAPSHOT_MANIFEST: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${version}/manifest-latest-verified.json`,
@@ -89,12 +92,12 @@ export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] {
 
 /**
  * This pipeline creates Kibana artifact snapshots for all open branches.
- * Should be triggered for all open branches in the versions.json: 7.x, 8.x
+ * Should be triggered for all open branches in the versions.json
  */
 export function getArtifactSnapshotPipelineTriggers() {
   // Trigger for all named branches
   const versions = getVersionsFile();
-  const targetVersions = [versions.prevMajors, versions.prevMinors, versions.current].flat();
+  const targetVersions = versions.versions;
 
   return targetVersions.map(({ branch }) => {
     return {
@@ -115,12 +118,14 @@ export function getArtifactSnapshotPipelineTriggers() {
 
 /**
  * This pipeline creates Kibana artifacts for branches that are not the current main.
- * Should be triggered for all open branches in the versions.json: 7.x, 8.x, but not main.
+ * Should be triggered for all open branches with a fixed version: not main and 8.x.
  */
 export function getArtifactStagingPipelineTriggers() {
   // Trigger for all branches, that are not current minor+major
   const versions = getVersionsFile();
-  const targetVersions = [versions.prevMajors, versions.prevMinors].flat();
+  const targetVersions = versions.versions.filter((version) =>
+    Boolean(version.branch.match(/[0-9]{1,2}\.[0-9]{1,2}/))
+  );
 
   return targetVersions.map(({ branch }) => {
     return {
@@ -142,13 +147,15 @@ export function getArtifactStagingPipelineTriggers() {
 /**
  * This pipeline checks if there are any changes in the incorporated $BEATS_MANIFEST_LATEST_URL (beats version)
  * and triggers a staging artifact build.
- * Should be triggered only for the active minor versions that are not `main` and not `7.17`.
+ * Should be triggered for all open branches with a fixed version excluding 7.17: not main, 7.17 and 8.x.
  *
  * TODO: we could basically do the check logic of .buildkite/scripts/steps/artifacts/trigger.sh in here, and remove kibana-artifacts-trigger
  */
 export function getArtifactBuildTriggers() {
   const versions = getVersionsFile();
-  const targetVersions = versions.prevMinors;
+  const targetVersions = versions.prevMajors.filter((version) =>
+    Boolean(version.branch.match(/[0-9]{1,2}\.[0-9]{1,2}/))
+  );
 
   return targetVersions.map(
     ({ branch }) =>

.buildkite/scripts/serverless/create_deploy_tag/info_sections/commit_info.ts

@@ -36,7 +36,7 @@ export async function getCurrentQARelease() {
   // @ts-ignore
   const fileContent = Buffer.from(releasesFile.data.content, 'base64').toString('utf8');
 
-  const sha = fileContent.match(`qa: "([a-z0-9]+)"`)?.[1];
+  const sha = fileContent.match(`qa-ds-1: "([a-z0-9]+)"`)?.[1];
 
   if (!sha) {
     throw new Error('Could not find QA hash in current releases file');