Merge remote-tracking branch 'elastic/master' into reduce-metadata-wr…

…ites-master

.ci/build.sh

@@ -1,16 +1,5 @@
 #!/bin/bash
 
-JAVA_HOME=${JAVA_HOME:-$HOME/.java/openjdk13}
-RUNTIME_JAVA_HOME=${RUNTIME_JAVA_HOME:-$HOME/.java/openjdk11}
-
-JAVA7_HOME=$HOME/.java/java7
-JAVA8_HOME=$HOME/.java/java8
-JAVA9_HOME=$HOME/.java/java9
-JAVA10_HOME=$HOME/.java/java10
-JAVA11_HOME=$HOME/.java/java11
-JAVA12_HOME=$HOME/.java/openjdk12
-JAVA13_HOME=$HOME/.java/openjdk13
-
 # drop page cache and kernel slab objects on linux
 [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
@@ -36,7 +25,6 @@ if pwd | grep -v -q ^/dev/shm ; then
    MAX_WORKERS=$(($MAX_WORKERS*2/3))
 fi
 
-export GRADLE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m"
 set -e
 ./gradlew --parallel --scan \
   -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \

.ci/jobs.t/defaults.yml

@@ -36,6 +36,20 @@
       - ansicolor
       - timestamps
       - gradle-build-scan
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            WORKSPACE=/dev/shm/$JOB_NAME
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA7_HOME=$HOME/.java/java7
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA9_HOME=$HOME/.java/java9
+            JAVA10_HOME=$HOME/.java/java10
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA12_HOME=$HOME/.java/openjdk12
+            JAVA13_HOME=$HOME/.java/openjdk13
+            GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m
     properties:
         - github:
             url: https://github.com/elastic/elasticsearch/
@@ -76,4 +90,4 @@
 
 
 
-            
+

.ci/jobs.t/elastic+elasticsearch+{branch}+docker-test-ramdisk.yml

@@ -0,0 +1,11 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+docker-test-ramdisk
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+docker-test
+    display-name: "elastic / elasticsearch # %BRANCH% - docker test ramdisk (experimental)"
+    description: "Experimental test job to shake out issues with Docker in CI\n"
+    node: "(debian-10 || sles-12 || debian-8 || sles-15) && immutable"
+    triggers: []
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
+          .ci/build.sh -Dbwc.checkout.align=true -p test/fixtures composeUp

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.4.0-snapshot-08b8d116f8f
+lucene            = 8.4.0
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.1+9

distribution/docker/build.gradle

@@ -22,31 +22,29 @@ dependencies {
   restSpec project(':rest-api-spec')
 }
 
-ext.expansions = { oss, ubi, local ->
+ext.expansions = { oss, local ->
   final String classifier = 'linux-x86_64'
   final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz"
   return [
-    'base_image'          : ubi ? 'registry.access.redhat.com/ubi7/ubi-minimal:7.7' : 'centos:7',
     'build_date'          : BuildParams.buildDate,
     'elasticsearch'       : elasticsearch,
     'git_revision'        : BuildParams.gitRevision,
     'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
-    'package_manager'     : ubi ? 'microdnf' : 'yum',
     'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
     'version'             : VersionProperties.elasticsearch
   ]
 }
 
-private static String buildPath(final boolean oss, final boolean ubi) {
-  return "build/${oss ? 'oss-' : ''}${ubi ? 'ubi-' : ''}docker"
+private static String buildPath(final boolean oss) {
+  return "build/${oss ? 'oss-' : ''}docker"
 }
 
-private static String taskName(final String prefix, final boolean oss, final boolean ubi, final String suffix) {
-  return "${prefix}${oss ? 'Oss' : ''}${ubi ? 'Ubi' : ''}${suffix}"
+private static String taskName(final String prefix, final boolean oss, final String suffix) {
+  return "${prefix}${oss ? 'Oss' : ''}${suffix}"
 }
 
 project.ext {
-  dockerBuildContext = { boolean oss, boolean ubi, boolean local ->
+  dockerBuildContext = { boolean oss, boolean local ->
     copySpec {
       into('bin') {
         from project.projectDir.toPath().resolve("src/docker/bin")
@@ -64,20 +62,20 @@ project.ext {
       }
 
       from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
-        expand(expansions(oss, ubi, local))
+        expand(expansions(oss, local))
       }
     }
   }
 }
 
-void addCopyDockerContextTask(final boolean oss, final boolean ubi) {
-  task(taskName("copy", oss, ubi, "DockerContext"), type: Sync) {
-    expansions(oss, ubi, true).findAll { it.key != 'build_date' }.each { k, v ->
+void addCopyDockerContextTask(final boolean oss) {
+  task(taskName("copy", oss, "DockerContext"), type: Sync) {
+    expansions(oss, true).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-    into buildPath(oss, ubi)
+    into buildPath(oss)
 
-    with dockerBuildContext(oss, ubi, true)
+    with dockerBuildContext(oss, true)
 
     if (oss) {
       from configurations.ossDockerSource
@@ -149,25 +147,25 @@ task integTest(type: Test) {
 
 check.dependsOn integTest
 
-void addBuildDockerImage(final boolean oss, final boolean ubi) {
-  final Task buildDockerImageTask = task(taskName("build", oss, ubi, "DockerImage"), type: LoggedExec) {
-    inputs.files(tasks.named(taskName("copy", oss, ubi, "DockerContext")))
+void addBuildDockerImage(final boolean oss) {
+  final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
+    inputs.files(tasks.named(taskName("copy", oss, "DockerContext")))
     List<String> tags
     if (oss) {
       tags = [
-        "docker.elastic.co/elasticsearch/elasticsearch-oss${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}",
-        "elasticsearch-oss${ubi ? '-ubi7' : ''}:test"
+        "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}",
+        "elasticsearch-oss:test"
       ]
     } else {
       tags = [
-        "elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}",
-        "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}",
-        "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}-full:${VersionProperties.elasticsearch}",
-        "elasticsearch${ubi ? '-ubi7' : ''}:test",
+        "elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}",
+        "elasticsearch:test",
       ]
     }
     executable 'docker'
-    final List<String> dockerArgs = ['build', buildPath(oss, ubi), '--pull', '--no-cache']
+    final List<String> dockerArgs = ['build', buildPath(oss), '--pull', '--no-cache']
     for (final String tag : tags) {
       dockerArgs.add('--tag')
       dockerArgs.add(tag)
@@ -184,10 +182,8 @@ void addBuildDockerImage(final boolean oss, final boolean ubi) {
 }
 
 for (final boolean oss : [false, true]) {
-  for (final boolean ubi : [false, true]) {
-    addCopyDockerContextTask(oss, ubi)
-    addBuildDockerImage(oss, ubi)
-  }
+  addCopyDockerContextTask(oss)
+  addBuildDockerImage(oss)
 }
 
 // We build the images used in compose locally, but the pull command insists on using a repository
@@ -206,26 +202,25 @@ subprojects { Project subProject ->
     apply plugin: 'distribution'
 
     final boolean oss = subProject.name.contains('oss-')
-    final boolean ubi = subProject.name.contains('ubi-')
 
-    def exportTaskName = taskName("export", oss, ubi, "DockerImage")
-    def buildTaskName = taskName("build", oss, ubi, "DockerImage")
-    def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
+    def exportTaskName = taskName("export", oss, "DockerImage")
+    def buildTaskName = taskName("build", oss, "DockerImage")
+    def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar"
 
     final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) {
       executable 'docker'
       outputs.file(tarFile)
       args "save",
         "-o",
         tarFile,
-        "elasticsearch${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}:test"
+        "elasticsearch${oss ? '-oss' : ''}:test"
     }
 
     exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName))
 
     artifacts.add('default', file(tarFile)) {
       type 'tar'
-      name "elasticsearch${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}"
+      name "elasticsearch${oss ? '-oss' : ''}"
       builtBy exportTaskName
     }
 

distribution/docker/docker-build-context/build.gradle

@@ -5,7 +5,7 @@ task buildDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch"
-  with dockerBuildContext(false, false, false)
+  with dockerBuildContext(false, false)
 }
 
 assemble.dependsOn buildDockerBuildContext

distribution/docker/oss-docker-build-context/build.gradle

@@ -5,7 +5,7 @@ task buildOssDockerBuildContext(type: Tar) {
   compression = Compression.GZIP
   archiveClassifier = "docker-build-context"
   archiveBaseName = "elasticsearch-oss"
-  with dockerBuildContext(true, false, false)
+  with dockerBuildContext(true, false)
 }
 
 assemble.dependsOn buildOssDockerBuildContext

distribution/docker/src/docker/Dockerfile

@@ -11,11 +11,11 @@
 # Set gid=0 and make group perms==owner perms
 ################################################################################
 
-FROM ${base_image} AS builder
+FROM centos:7 AS builder
 
-RUN for iter in {1..10}; do ${package_manager} update --setopt=tsflags=nodocs -y && \
-    ${package_manager} install --setopt=tsflags=nodocs -y gzip shadow-utils tar && \
-    ${package_manager} clean all && exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10; done; \
+RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
+    yum install --setopt=tsflags=nodocs -y gzip shadow-utils tar && \
+    yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \
     (exit \$exit_code)
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
@@ -41,13 +41,13 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 # Add entrypoint
 ################################################################################
 
-FROM ${base_image}
+FROM centos:7
 
 ENV ELASTIC_CONTAINER true
 
-RUN for iter in {1..10}; do ${package_manager} update --setopt=tsflags=nodocs -y && \
-    ${package_manager} install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \
-    ${package_manager} clean all && exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10; done; \
+RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \
+    yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \
+    yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \
     (exit \$exit_code)
 
 RUN groupadd -g 1000 elasticsearch && \

docs/painless/painless-contexts/painless-reindex-context.asciidoc

@@ -10,7 +10,7 @@ reindexed into a target index.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`ctx['_op']` (`String`)::
+`ctx['op']` (`String`)::
         The name of the operation.
 
 {ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`)::
@@ -34,7 +34,7 @@ reindexed into a target index.
 
 *Side Effects*
 
-`ctx['_op']`::
+`ctx['op']`::
         Use the default of `index` to update a document. Set to `none` to
         specify no operation or `delete` to delete the current document from
         the index.
@@ -65,4 +65,4 @@ reindexed into a target index.
 
 *API*
 
-The standard <<painless-api-reference-shared, Painless API>> is available.
+The standard <<painless-api-reference-shared, Painless API>> is available.

docs/painless/painless-contexts/painless-update-by-query-context.asciidoc

@@ -11,7 +11,7 @@ result of query.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`ctx['_op']` (`String`)::
+`ctx['op']` (`String`)::
         The name of the operation.
 
 {ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
@@ -35,7 +35,7 @@ result of query.
 
 *Side Effects*
 
-`ctx['_op']`::
+`ctx['op']`::
         Use the default of `index` to update a document. Set to `none` to
         specify no operation or `delete` to delete the current document from
         the index.
@@ -91,4 +91,4 @@ POST /seats/_update_by_query
     }
 }
 --------------------------------------------------
-// TEST[setup:seats]
+// TEST[setup:seats]

docs/painless/painless-contexts/painless-update-context.asciidoc

@@ -9,7 +9,7 @@ add, modify, or delete fields within a single document.
 `params` (`Map`, read-only)::
         User-defined parameters passed in as part of the query.
 
-`ctx['_op']` (`String`)::
+`ctx['op']` (`String`)::
         The name of the operation.
 
 {ref}/mapping-routing-field.html[`ctx['_routing']`] (`String`, read-only)::
@@ -36,7 +36,7 @@ add, modify, or delete fields within a single document.
 
 *Side Effects*
 
-`ctx['_op']`::
+`ctx['op']`::
         Use the default of `index` to update a document. Set to `none` to
         specify no operation or `delete` to delete the current document from
         the index.
@@ -75,4 +75,4 @@ POST /seats/_update/3
     }
 }
 --------------------------------------------------
-// TEST[setup:seats]
+// TEST[setup:seats]