Merge branch 'upstream/master' into update-filter-improve

* upstream/master: (319 commits)
  Fix typo (elastic#80925)
  Increase docker compose timeouts for CI builds
  TSDB: fix error without feature flag (elastic#80945)
  [DOCS] Relocate `index.mapping.dimension_fields.limit` setting docs (elastic#80964)
  Explicit write methods for always-missing values (elastic#80958)
  TSDB: move TimeSeriesModeIT to yaml tests (elastic#80933)
  [ML] Removing temporary debug (elastic#80956)
  Remove unused ConnectTransportException#node (elastic#80944)
  Reinterpret dots in field names as object structure (elastic#79922)
  Remove obsolete typed legacy index templates (elastic#80937)
  Remove unnecessary shuffle in unassigned shards allocation. (elastic#65172)
  TSDB: Tests for nanosecond timeprecision timestamp just beyond the limit (elastic#80932)
  Cleanup SLM History Item .equals (elastic#80938)
  Rework breaking changes for new structure (elastic#80907)
  [DOCS] Fix elasticsearch-reset-password typo (elastic#80919)
  [ML] No need to use parent task client when internal infer delegates (elastic#80905)
  Fix shadowed vars pt6 (elastic#80899)
  add ignore info (elastic#80924)
  Fix several potential circuit breaker leaks in Aggregators (elastic#79676)
  Extract more standard metadata from binary files (elastic#78754)
  ...

.ci/bwcVersions

@@ -44,6 +44,7 @@ BWC_VERSION:
   - "7.15.0"
   - "7.15.1"
   - "7.15.2"
+  - "7.15.3"
   - "7.16.0"
   - "8.0.0"
   - "8.1.0"

.ci/jobs.t/defaults.yml

@@ -53,6 +53,7 @@
           url: https://github.com/elastic/elasticsearch/
       - inject:
           properties-content: |
+            COMPOSE_HTTP_TIMEOUT=120
             JOB_BRANCH=%BRANCH%
             HOME=$JENKINS_HOME
             GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml

@@ -15,7 +15,6 @@
           values:
             - centos-7-packaging
             - centos-8-packaging
-            - debian-9-packaging
             - debian-10-packaging
             - debian-11-packaging
             - opensuse-15-1-packaging
@@ -26,6 +25,9 @@
             - ubuntu-18.04-packaging
             - ubuntu-20.04-packaging
             - rocky-linux-8-packaging
+            - rhel-7-packaging
+            - rhel-8-packaging
+            - almalinux-8-packaging
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml

@@ -25,7 +25,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -24,7 +24,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/jdk11
-            JAVA15_HOME=$HOME/.java/jdk15
+            JAVA16_HOME=$HOME/.java/jdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml

@@ -11,7 +11,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -15,7 +15,7 @@
           values:
             - "centos-7&&immutable"
             - "centos-8&&immutable"
-            - "debian-9&&immutable"
+            - "amazon&&immutable"
             - "debian-10&&immutable"
             - "debian-11&&immutable"
             - "opensuse-15-1&&immutable"
@@ -26,14 +26,17 @@
             - "ubuntu-18.04&&immutable"
             - "ubuntu-20.04&&immutable"
             - "rocky-linux-8&&immutable"
+            - "rhel-7&&immutable"
+            - "rhel-8&&immutable"
+            - "almalinux-8&&immutable"
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml

@@ -35,7 +35,7 @@
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
             JAVA11_HOME=$USERPROFILE\\.java\\java11
-            JAVA15_HOME=$USERPROFILE\\.java\\openjdk15
+            JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
       - batch: |
           del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
           mkdir %USERPROFILE%\.gradle\init.d

.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml

@@ -12,7 +12,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/bin/bash
           # Configure a dm-crypt volume backed by a file

.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml

@@ -1,7 +1,8 @@
 ---
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+release-tests
-    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+release-tests
+    # Don't use ramdisk since this build generates lots of large artifacts and results in oomkiller issues
+    # workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+release-tests
     display-name: "elastic / elasticsearch # %BRANCH% - release tests"
     description: "Release version tests for the Elasticsearch %BRANCH% branch.\n"
     node: "general-purpose && docker"
@@ -12,7 +13,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
 

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml

@@ -31,7 +31,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed bwcTestSnapshots

.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml

@@ -0,0 +1,50 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+cloud-deploy"
+    display-name: "elastic / elasticsearch - pull request cloud-deploy"
+    description: "Testing of Elasticsearch pull requests - cloud-deploy"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+cloud-deploy"
+    node: "general-purpose && docker"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/cloud-deploy.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/cloud-deploy
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'cloud-deploy'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh buildCloudDockerImage
+      - shell: |
+          #!/bin/bash
+          set +x
+          VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+          export VAULT_TOKEN
+          export DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/elasticsearch-ci/prod_docker_registry_credentials)"
+          export DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/elasticsearch-ci/prod_docker_registry_credentials)"
+          export ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
+          export DOCKER_TAG=docker.elastic.co/elasticsearch-ci/elasticsearch-cloud:${ES_VERSION}-${ghprbActualCommit:0:7}
+          docker tag elasticsearch-cloud:test $DOCKER_TAG
+          echo $DOCKER_REGISTRY_PASSWORD | docker login -u $DOCKER_REGISTRY_USERNAME --password-stdin docker.elastic.co
+          unset VAULT_TOKEN DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
+          set -x
+          docker push $DOCKER_TAG

.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml

@@ -0,0 +1,34 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+example-plugins"
+    display-name: "elastic / elasticsearch - pull request example-plugins"
+    description: "Testing of Elasticsearch pull requests - example-plugins"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+example-plugins"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/example-plugins.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/example-plugins
+          cancel-builds-on-update: true
+          white-list-labels:
+            - ':Delivery/Build'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          cd plugins/examples
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dorg.gradle.jvmargs=-Xmx8g build --include-build $WORKSPACE

.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml

@@ -44,7 +44,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed v$BWC_VERSION#bwcTest

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -47,7 +47,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION

.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml

@@ -33,7 +33,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart1

.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml

@@ -33,7 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
-            JAVA15_HOME=$USERPROFILE\\.java\\openjdk15
+            JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart1
       - batch: |
           del /f /s /q %USERPROFILE%\.gradle\init.d\*.*

.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml

@@ -33,7 +33,7 @@
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart2

.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml

@@ -33,7 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
-            JAVA15_HOME=$USERPROFILE\\.java\\openjdk15
+            JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart2
       - batch: |
           del /f /s /q %USERPROFILE%\.gradle\init.d\*.*

.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml

@@ -3,7 +3,9 @@
     name: "elastic+elasticsearch+pull-request+release-tests"
     display-name: "elastic / elasticsearch - pull request release-tests"
     description: "Testing of Elasticsearch pull requests - release-tests"
-    workspace: "/dev/shm/elastic+elasticsearch+pull-request+release-tests"
+    # Don't use ramdisk since this build generates lots of large artifacts and results in oomkiller issues
+    # workspace: "/dev/shm/elastic+elasticsearch+pull-request+release-tests"
+    node: "general-purpose && docker"
     scm:
       - git:
           refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
@@ -33,7 +35,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
 

.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml

@@ -34,7 +34,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed checkRestCompat

.ci/packer_cache.sh

@@ -19,7 +19,7 @@ if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
   ## On ARM we use a different properties file for setting java home
   ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch
   source $(dirname "${SCRIPT}")/java-versions-aarch64.properties
-  export JAVA15_HOME="${HOME}"/.java/jdk15
+  export JAVA16_HOME="${HOME}"/.java/jdk16
 else
   source $(dirname "${SCRIPT}")/java-versions.properties
   ## We are caching BWC versions too, need these so we can build those
@@ -29,6 +29,7 @@ else
   export JAVA13_HOME="${HOME}"/.java/openjdk13
   export JAVA14_HOME="${HOME}"/.java/openjdk14
   export JAVA15_HOME="${HOME}"/.java/openjdk15
+  export JAVA16_HOME="${HOME}"/.java/openjdk16
 
   ## 6.8 branch is not referenced from any bwc project in master so we need to
   ## resolve its dependencies explicitly

.ci/templates.t/generic-gradle-unix.yml

@@ -11,7 +11,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args}

.ci/templates.t/matrix-gradle-unix.yml

@@ -26,7 +26,7 @@
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args}

.ci/templates.t/pull-request-gradle-unix.yml

@@ -30,7 +30,7 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA8_HOME=$HOME/.java/java8
             JAVA11_HOME=$HOME/.java/java11
-            JAVA15_HOME=$HOME/.java/openjdk15
+            JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
           $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args}

.git-blame-ignore-revs

@@ -3,7 +3,7 @@
 #
 # Use this file by running:
 #
-#     git blame --ignore-revs-file=.git-blame-ignore-rev <file>
+#     git blame --ignore-revs-file=.git-blame-ignore-revs <file>
 #
 # or by configuring `blame.ignoreRevsFile`. The latter ought to also work
 # with IDEs such as IntelliJ.

CONTRIBUTING.md

@@ -120,8 +120,7 @@ JDK 16 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 11 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 11.
 
-> Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
-and `JAVA11_HOME`, `JAVA12_HOME`, `JAVA13_HOME`, `JAVA14_HOME`, and `JAVA15_HOME`
+> Note: It is also required to have `JAVA8_HOME`, `JAVA11_HOME`, and `JAVA17_HOME`
 available so that the tests can pass.
 
 Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle