Merge branch 'main' into fix/97150-test-downsample-datastream-index-p…

…attern

.buildkite/hooks/pre-command

@@ -92,7 +92,7 @@ fi
 
 # Initialize the build scan and gobld annotations with empty/open <details> tags
 # This ensures that they are collapsible when they get appended to
-if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* ]]; then
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
   cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
 <details>
 

.buildkite/pipelines/intake.template.yml

@@ -14,23 +14,31 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part2
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part3
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:

.buildkite/pipelines/intake.yml

@@ -15,23 +15,31 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part2
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part3
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
@@ -40,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0", "8.12.0"]
+            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic-packaging.yml

@@ -20,6 +20,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9
@@ -1056,6 +1057,38 @@ steps:
         env:
           BWC_VERSION: 7.17.14
 
+      - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
+
+      - label: "{{matrix.image}} / 7.17.16 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.16
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.16
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1648,6 +1681,22 @@ steps:
         env:
           BWC_VERSION: 8.10.3
 
+      - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300
@@ -1664,6 +1713,38 @@ steps:
         env:
           BWC_VERSION: 8.11.0
 
+      - label: "{{matrix.image}} / 8.11.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.1
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.1
+
+      - label: "{{matrix.image}} / 8.11.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.2
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -2,7 +2,7 @@ steps:
   - group: platform-support-unix
     steps:
       - label: "{{matrix.image}} / platform-support-unix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
         timeout_in_minutes: 420
         matrix:
           setup:
@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9
@@ -46,7 +47,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: gcp
@@ -70,7 +70,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: aws

.buildkite/pipelines/periodic.bwc.template.yml

@@ -4,7 +4,7 @@
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: $BWC_VERSION

.buildkite/pipelines/periodic.template.yml

@@ -54,7 +54,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
@@ -82,14 +82,14 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
@@ -180,3 +180,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
+    if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+\$/
+  - label: Check branch consistency
+    command: .ci/scripts/run-gradle.sh branchConsistency
+    timeout_in_minutes: 15
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-2
+  - label: Check branch protection rules
+    command: .buildkite/scripts/branch-protection.sh
+    timeout_in_minutes: 5