Merge branch 'main' into rollup-support-cancel

* main: (1300 commits)
  update c2id/c2id-server-demo docker image to support ARM (elastic#91144)
  Allow legacy index settings on legacy indices (elastic#90264)
  Skip prevoting if single-node discovery (elastic#91255)
  Chunked encoding for snapshot status API (elastic#90801)
  Allow different decay values depending on the score function (elastic#91195)
  Fix handling indexed envelopes crossing the dateline in mvt API (elastic#91105)
  Ensure cleanups succeed in JoinValidationService (elastic#90601)
  Add overflow behaviour test for RecyclerBytesStreamOutput (elastic#90638)
  More actionable error for ancient indices (elastic#91243)
  Fix APM configuration file delete (elastic#91058)
  Clean up handshake test class (elastic#90966)
  Improve H3#hexRing logic and add H3#areNeighborCells method (elastic#91140)
  Restrict direct use of `ApplicationPrivilege` constructor (elastic#91176)
  [ML] Allow NLP truncate option to be updated when span is set (elastic#91224)
  Support multi-intersection for FieldPermissions (elastic#91169)
  Support intersecting multi-sets of queries with DocumentPermissions (elastic#91151)
  Ensure TermsEnum action works correctly with API keys (elastic#91170)
  Fix NPE in auditing authenticationSuccess for non-existing run-as user (elastic#91171)
  Ensure PKI's delegated_by_realm metadata respect run-as (elastic#91173)
  [ML] Update API documentation for anomaly score explanation (elastic#91177)
  ...

# Conflicts:
#	x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupIndexerAction.java
#	x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/RollupActionSingleNodeTests.java

.backportrc.json

@@ -1,17 +1,9 @@
 {
-  "upstream": "elastic/elasticsearch",
-  "targetBranchChoices": [
-    "master",
-    "8.3",
-    "8.2",
-    "8.1",
-    "8.0",
-    "7.17",
-    "6.8"
-  ],
-  "targetPRLabels": ["backport"],
-  "branchLabelMapping": {
-    "^v8.4.0$": "master",
-    "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$": "$1.$2"
+  "upstream" : "elastic/elasticsearch",
+  "targetBranchChoices" : [ "main", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetPRLabels" : [ "backport" ],
+  "branchLabelMapping" : {
+    "^v8.6.0$" : "main",
+    "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
-}
+}

.ci/bwcVersions

@@ -55,6 +55,8 @@ BWC_VERSION:
   - "7.17.4"
   - "7.17.5"
   - "7.17.6"
+  - "7.17.7"
+  - "7.17.8"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -70,3 +72,9 @@ BWC_VERSION:
   - "8.3.2"
   - "8.3.3"
   - "8.4.0"
+  - "8.4.1"
+  - "8.4.2"
+  - "8.4.3"
+  - "8.5.0"
+  - "8.5.1"
+  - "8.6.0"

.ci/jobs.t/defaults.yml

@@ -56,8 +56,8 @@
             COMPOSE_HTTP_TIMEOUT=120
             JOB_BRANCH=%BRANCH%
             HOME=$JENKINS_HOME
-            GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
-            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW=./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
     publishers:
 # Disable Junit publisher for now since it's causing OOME on our Jenkins instance
 #      - junit:

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part3.yml

@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part3
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 3"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 3.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3"

.ci/jobs.t/elastic+elasticsearch+intake.yml

@@ -42,6 +42,10 @@
               kill-phase-on: NEVER
               current-parameters: true
               git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part3
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
             - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc-snapshots
               kill-phase-on: NEVER
               current-parameters: true

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -14,7 +14,8 @@
           name: os
           values:
             - "centos-7&&immutable"
-            - "amazon&&immutable"
+            - "amazon-2&&immutable"
+            - "amazon-2022&&immutable"
             - "debian-10&&immutable"
             - "debian-11&&immutable"
             - "opensuse-15-1&&immutable"

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml

@@ -27,6 +27,7 @@
           values:
             - 'checkPart1'
             - 'checkPart2'
+            - 'checkPart3'
             - 'bwcTestSnapshots'
             - 'checkRestCompat'
     builders:

.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml

@@ -1,7 +1,6 @@
 ---
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+ear
-    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+ear
     display-name: "elastic / elasticsearch # %BRANCH% - encryption at rest"
     description: "The Elasticsearch %BRANCH% branch encryption at rest compatibility tests.\n\n"
     node: packaging-large

.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml

@@ -1,9 +1,42 @@
 ---
-jjbb-template: matrix-gradle-unix.yml
-vars:
-  - job-name: elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix
-  - job-display-name: "elastic / elasticsearch # %BRANCH% - java fips compatibility matrix"
-  - job-description: "Testing of the Elasticsearch %BRANCH% branch java FIPS compatibility matrix.\n"
-  - matrix-yaml-file: ".ci/matrix-runtime-javas-fips.yml"
-  - matrix-variable: ES_RUNTIME_JAVA
-  - gradle-args: "-Dbwc.checkout.align=true -Dtests.fips.enabled=true check"
+- job:
+    name: "elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix"
+    display-name: "elastic / elasticsearch # %BRANCH% - java fips compatibility matrix"
+    description: "Testing of the Elasticsearch %BRANCH% branch java FIPS compatibility matrix.\n"
+    project-type: matrix
+    child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      - axis:
+          type: yaml
+          filename: ".ci/matrix-runtime-javas-fips.yml"
+          name: "ES_RUNTIME_JAVA"
+      # We shred out these jobs to avoid running out of memory given since we use a ramdisk workspace
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'checkPart3'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA16_HOME=$HOME/.java/openjdk16
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $GRADLE_TASK

.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml

@@ -26,6 +26,7 @@
           values:
             - 'checkPart1'
             - 'checkPart2'
+            - 'checkPart3'
             - 'bwcTestSnapshots'
             - 'checkRestCompat'
     builders:

.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+java-preview-features
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features.yml

@@ -0,0 +1,38 @@
+---
+- job:
+    name: "elastic+elasticsearch+%BRANCH%+periodic+java-preview-features"
+    display-name: "elastic / elasticsearch # %BRANCH% - java preview features"
+    description: "Testing of the Elasticsearch %BRANCH% branch with java preview features enabled.\n"
+    project-type: matrix
+    child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-preview-features
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      # We shred out these jobs to avoid running out of memory given since we use a ramdisk workspace
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'checkPart3'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/openjdk19
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA16_HOME=$HOME/.java/openjdk16
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dtests.jvm.argline="--enable-preview" -Dbwc.checkout.align=true $GRADLE_TASK

.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml

@@ -22,8 +22,14 @@
           export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
 
           mkdir -p ${BEATS_DIR}
-          curl -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
 
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \
+          # Fetch ML artifacts
+          export ML_IVY_REPO=$(mktemp -d)
+          mkdir -p ${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-deps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-deps.zip
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-nodeps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-nodeps.zip
+
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
             -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml

@@ -0,0 +1,16 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+    display-name: "elastic / elasticsearch # %BRANCH% - single processor node tests"
+    description: "Testing with node.processors set to '1' for the Elasticsearch %BRANCH% branch.\n"
+    node: "general-purpose && docker"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H * * *"

.ci/jobs.t/elastic+elasticsearch+periodic+snyk-dependency-monitoring.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+snyk-dependency-monitoring
+    display-name: "elastic / elasticsearch # %BRANCH% - snyk dependency monitoring"
+    description: "Publishing of the Elasticsearch %BRANCH% dependencies graph to snyk dependency monitoring"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          set -euo pipefail
+          set +x
+          VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+          export VAULT_TOKEN
+          export SNYK_TOKEN=$(vault read -field=token secret/elasticsearch-ci/snyk)
+          unset VAULT_TOKEN
+          set -x
+          $WORKSPACE/.ci/scripts/run-gradle.sh uploadSnykDependencyGraph

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml

@@ -25,8 +25,6 @@
           black-list-labels:
             - '>test-mute'
             - 'test-full-bwc'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     axes:
       - axis:
           type: slave

.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml

@@ -22,8 +22,6 @@
             - ^docs/.*
           black-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml

@@ -20,7 +20,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml

@@ -23,8 +23,6 @@
             - build-tools/.*
             - build-tools-internal/.*
             - plugins/examples/.*
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -21,7 +21,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:
@@ -32,7 +31,7 @@
           type: label-expression
           name: os
           values:
-            - rocky-linux-8-packaging
+            - rhel-8-packaging
             - ubuntu-20.04-packaging
       - axis:
           type: user-defined

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -24,7 +24,6 @@
           black-list-target-branches:
             - 7.17
             - 7.16
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -22,7 +22,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           white-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml

@@ -0,0 +1,40 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+part-3-fips"
+    display-name: "elastic / elasticsearch - pull request part-3 fips"
+    description: "Testing of Elasticsearch pull requests - part-3 fips"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+part-3-fips"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/part-3-fips.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/part-3-fips
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+            - 7.17
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'Team:Security'
+          black-list-labels:
+            - '>test-mute'
+    builders:
+      - inject:
+          # Use FIPS-specific Java versions
+          properties-file: '.ci/java-versions-fips.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA16_HOME=$HOME/.java/openjdk16
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart3