Merge branch 'main' into version/assert-sysindex-version-hashes-match

.buildkite/pipelines/periodic.yml

@@ -1,4 +1,3 @@
-# This file is auto-generated. See .buildkite/pipelines/periodic.yml
 # This file is auto-generated. See .buildkite/pipelines/periodic.template.yml
 steps:
   - group: bwc

build.gradle

@@ -64,6 +64,17 @@ ext.testArtifact = { p, String name = "test" ->
   };
 }
 
+class StepExpansion {
+  String templatePath
+  List<Version> versions
+  String variable
+}
+
+class ListExpansion {
+  List<Version> versions
+  String variable
+}
+
 tasks.register("updateCIBwcVersions") {
   def writeVersions = { File file, List<Version> versions ->
     file.text = ""
@@ -73,47 +84,60 @@ tasks.register("updateCIBwcVersions") {
     }
   }
 
-  def writeBuildkiteList = { String outputFilePath, String pipelineTemplatePath, List<Version> versions ->
+  def writeBuildkitePipeline = { String outputFilePath, String pipelineTemplatePath, List<ListExpansion> listExpansions, List<StepExpansion> stepExpansions = [] ->
     def outputFile = file(outputFilePath)
     def pipelineTemplate = file(pipelineTemplatePath)
 
-    def listString = "[" + versions.collect { "\"${it}\"" }.join(", ") + "]"
-    outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipelineTemplate.text.replaceAll('\\$BWC_LIST', listString)
-  }
+    def pipeline = pipelineTemplate.text
 
-  def writeBuildkiteSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
-    def outputFile = file(outputFilePath)
-    def pipelineTemplate = file(pipelineTemplatePath)
-    def stepTemplate = file(stepTemplatePath)
+    listExpansions.each { expansion ->
+      def listString = "[" + expansion.versions.collect { "\"${it}\"" }.join(", ") + "]"
+      pipeline = pipeline.replaceAll('\\$' + expansion.variable, listString)
+    }
 
-    def steps = ""
-    versions.each {
-      steps += "\n" + stepTemplate.text.replaceAll('\\$BWC_VERSION', it.toString())
+    stepExpansions.each { expansion ->
+      def steps = ""
+      expansion.versions.each {
+        steps += "\n" + file(expansion.templatePath).text.replaceAll('\\$BWC_VERSION', it.toString())
+      }
+      pipeline = pipeline.replaceAll(' *\\$' + expansion.variable, steps)
     }
 
-    outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipelineTemplate.text.replaceAll(' *\\$BWC_STEPS', steps)
+    outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline
+  }
+
+  // Writes a Buildkite pipelime from a template, and replaces $BWC_LIST with an array of versions
+  // Useful for writing a list of versions in a matrix configuration
+  def expandBwcList = { String outputFilePath, String pipelineTemplatePath, List<Version> versions ->
+    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")])
+  }
+
+  // Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version
+  // Useful when you need to configure more versions than are allowed in a matrix configuration
+  def expandBwcSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
+    writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [], [new StepExpansion(templatePath: stepTemplatePath, versions: versions, variable: "BWC_STEPS")])
   }
 
   doLast {
     writeVersions(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
     writeVersions(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
-    writeBuildkiteList(
+    expandBwcList(
       ".buildkite/pipelines/intake.yml",
       ".buildkite/pipelines/intake.template.yml",
       BuildParams.bwcVersions.unreleasedIndexCompatible
     )
-    writeBuildkiteSteps(
+    writeBuildkitePipeline(
       ".buildkite/pipelines/periodic.yml",
       ".buildkite/pipelines/periodic.template.yml",
-      ".buildkite/pipelines/periodic.bwc.template.yml",
-      BuildParams.bwcVersions.allIndexCompatible
+      [
+        new ListExpansion(versions: BuildParams.bwcVersions.unreleasedIndexCompatible, variable: "BWC_LIST"),
+      ],
+      [
+        new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: BuildParams.bwcVersions.allIndexCompatible, variable: "BWC_STEPS"),
+      ]
     )
-    writeBuildkiteList(
-      ".buildkite/pipelines/periodic.yml",
-      ".buildkite/pipelines/periodic.yml",
-      BuildParams.bwcVersions.unreleasedIndexCompatible
-    )
-    writeBuildkiteSteps(
+
+    expandBwcSteps(
       ".buildkite/pipelines/periodic-packaging.yml",
       ".buildkite/pipelines/periodic-packaging.template.yml",
       ".buildkite/pipelines/periodic-packaging.bwc.template.yml",

docs/changelog/105674.yaml

@@ -0,0 +1,6 @@
+pr: 105674
+summary: Health monitor concurrency fixes
+area: Health
+type: bug
+issues:
+ - 105065

docs/changelog/105691.yaml

@@ -0,0 +1,5 @@
+pr: 105691
+summary: "ES|QL: Disable optimizations that rely on Expression.nullable()"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/105757.yaml

@@ -0,0 +1,5 @@
+pr: 105757
+summary: Add pluggable `BuildVersion` in `NodeMetadata`
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/105893.yaml

@@ -0,0 +1,5 @@
+pr: 105893
+summary: Specialize serialization for `ArrayVectors`
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106031.yaml

@@ -0,0 +1,13 @@
+pr: 106031
+summary: Deprecate allowing `fields` in scenarios where it is ignored
+area: Mapping
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate allowing `fields` in scenarios where it is ignored
+  area: Mapping
+  details: The following mapped types have always ignored `fields` when using multi-fields.
+    This deprecation makes this clearer and we will completely disallow `fields` for
+    these mapped types in the future.
+  impact: "In the future, `join`, `aggregate_metric_double`, and `constant_keyword`,\
+    \ will all disallow supplying `fields` as a parameter in the mapping."

docs/changelog/106062.yaml

@@ -0,0 +1,6 @@
+pr: 106062
+summary: "During ML maintenance, reset jobs in the reset state without a corresponding\
+  \ task"
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/106102.yaml

@@ -0,0 +1,5 @@
+pr: 106102
+summary: Specialize serialization of array blocks
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106105.yaml

@@ -0,0 +1,5 @@
+pr: 106105
+summary: Respect --pass option in certutil csr mode
+area: TLS
+type: bug
+issues: []

docs/reference/cluster/nodes-stats.asciidoc

@@ -1792,14 +1792,14 @@ Total number of unallocated bytes in all file stores.
 `available`::
 (<<byte-units,byte value>>)
 Total disk space available to this Java virtual machine on all file
-stores. Depending on OS or process level restrictions, this might appear
+stores. Depending on OS or process level restrictions (e.g. XFS quotas), this might appear
 less than `free`. This is the actual amount of free disk
 space the {es} node can utilise.
 
 `available_in_bytes`::
 (integer)
 Total number of bytes available to this Java virtual machine on all file
-stores. Depending on OS or process level restrictions, this might appear
+stores. Depending on OS or process level restrictions (e.g. XFS quotas), this might appear
 less than `free_in_bytes`. This is the actual amount of free disk
 space the {es} node can utilise.
 =======