Merge remote-tracking branch 'es/7.x' into enrich-7.x

Author: martijnvg

build.gradle

@@ -527,9 +527,9 @@ allprojects {
   def checkPart2 = tasks.register('checkPart2')
   plugins.withId('lifecycle-base') {
       if (project.path.startsWith(":x-pack:")) {
-          checkPart1.configure { dependsOn 'check' }
-      } else {
           checkPart2.configure { dependsOn 'check' }
+      } else {
+          checkPart1.configure { dependsOn 'check' }
       }
   }
 }

buildSrc/build.gradle

@@ -71,12 +71,6 @@ sourceSets {
     minimumRuntime { }
 }
 
-allprojects {
-    apply plugin: 'java'
-    targetCompatibility = 11
-    sourceCompatibility = 11
-}
-
 configurations {
   reaper
 }
@@ -160,8 +154,8 @@ if (project != rootProject) {
   apply plugin: 'nebula.maven-scm'
 
   allprojects {
-    targetCompatibility = 11
-    sourceCompatibility = 11
+    targetCompatibility = 10
+    sourceCompatibility = 10
   }
 
   // groovydoc succeeds, but has some weird internal exception...

buildSrc/reaper/build.gradle

@@ -1,3 +1,5 @@
+apply plugin: 'java'
+
 jar {
     archiveName = "${project.name}.jar"
     manifest {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -848,10 +848,9 @@ class BuildPlugin implements Plugin<Project> {
                 test.exclude '**/*$*.class'
 
                 test.jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}",
-                        "-Xms${System.getProperty('tests.heap.size', '512m')}",
-                        '-XX:+HeapDumpOnOutOfMemoryError',
-                        "-XX:HeapDumpPath=$heapdumpDir"
+                        "-Xms${System.getProperty('tests.heap.size', '512m')}"
 
+                test.jvmArgumentProviders.add({ ['-XX:+HeapDumpOnOutOfMemoryError', "-XX:HeapDumpPath=$heapdumpDir"] } as CommandLineArgumentProvider)
 
                 if (System.getProperty('tests.jvm.argline')) {
                     test.jvmArgs System.getProperty('tests.jvm.argline').split(" ")
@@ -862,8 +861,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 // we use './temp' since this is per JVM and tests are forbidden from writing to CWD
-                test.systemProperties 'gradle.dist.lib': new File(project.class.location.toURI()).parent,
-                        'java.io.tmpdir': './temp',
+                test.systemProperties 'java.io.tmpdir': './temp',
                         'java.awt.headless': 'true',
                         'tests.gradle': 'true',
                         'tests.artifact': project.name,
@@ -879,6 +877,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 // don't track these as inputs since they contain absolute paths and break cache relocatability
+                nonInputProperties.systemProperty('gradle.dist.lib', new File(project.class.location.toURI()).parent)
                 nonInputProperties.systemProperty('gradle.worker.jar', "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar")
                 nonInputProperties.systemProperty('gradle.user.home', project.gradle.getGradleUserHomeDir())
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -237,16 +237,12 @@ class PrecommitTasks {
         // on them. We also want `precommit` to depend on `checkstyle`.
         project.pluginManager.apply('checkstyle')
         project.checkstyle {
-            config = project.resources.text.fromFile(checkstyleConf, 'UTF-8')
-            configProperties = [
-                    suppressions: checkstyleSuppressions
-            ]
+            configDir = checkstyleDir
             toolVersion = CHECKSTYLE_VERSION
         }
 
         project.tasks.withType(Checkstyle).configureEach { task ->
             task.dependsOn(copyCheckstyleConf)
-            task.inputs.file(checkstyleSuppressions)
             task.reports {
                 html.enabled false
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -24,13 +24,15 @@
 import org.gradle.api.logging.Logger;
 import org.gradle.internal.jvm.Jvm;
 
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -68,8 +70,8 @@ public void registerPid(String serviceId, long pid) {
     public void registerCommand(String serviceId, String... command) {
         ensureReaperStarted();
 
-        try {
-            Files.writeString(getCmdFile(serviceId), String.join(" ", command));
+        try (FileWriter writer = new FileWriter(getCmdFile(serviceId).toFile())) {
+            writer.write(String.join(" ", command));
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -143,7 +145,7 @@ private Path locateReaperJar() {
 
             if (matcher.matches()) {
                 String path = matcher.group(1);
-                return Path.of(
+                return Paths.get(
                     OS.<String>conditional()
                         .onWindows(() -> path.substring(1))
                         .onUnix(() -> path)

buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java

@@ -7,7 +7,6 @@
 import org.gradle.api.Project;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.internal.jvm.Jvm;
-import org.gradle.process.ExecResult;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
@@ -16,18 +15,19 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
     private static Integer _defaultParallel = null;
@@ -46,8 +46,6 @@ public void apply(Project project) {
         File compilerJavaHome = findCompilerJavaHome();
         File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome);
 
-        Object gitRevisionResolver = createGitRevisionResolver(project);
-
         final List<JavaHome> javaVersions = new ArrayList<>();
         for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) {
             if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) {
@@ -95,7 +93,7 @@ public void apply(Project project) {
             ext.set("minimumCompilerVersion", minimumCompilerVersion);
             ext.set("minimumRuntimeVersion", minimumRuntimeVersion);
             ext.set("gradleJavaVersion", Jvm.current().getJavaVersion());
-            ext.set("gitRevision", gitRevisionResolver);
+            ext.set("gitRevision", gitRevision(project));
             ext.set("buildDate", ZonedDateTime.now(ZoneOffset.UTC));
         });
     }
@@ -206,35 +204,65 @@ private static int findDefaultParallel(Project project) {
         return _defaultParallel;
     }
 
-    private Object createGitRevisionResolver(final Project project) {
-        return new Object() {
-            private final AtomicReference<String> gitRevision = new AtomicReference<>();
-
-            @Override
-            public String toString() {
-                if (gitRevision.get() == null) {
-                    final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-                    final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
-                    final ExecResult result = project.exec(spec -> {
-                        spec.setExecutable("git");
-                        spec.setArgs(Arrays.asList("rev-parse", "HEAD"));
-                        spec.setStandardOutput(stdout);
-                        spec.setErrorOutput(stderr);
-                        spec.setIgnoreExitValue(true);
-                    });
-
-                    final String revision;
-                    if (result.getExitValue() != 0) {
-                        revision = "unknown";
-                    } else {
-                        revision = stdout.toString(UTF_8).trim();
-                    }
-                    this.gitRevision.compareAndSet(null, revision);
+    private String gitRevision(final Project project) {
+        try {
+            /*
+             * We want to avoid forking another process to run git rev-parse HEAD. Instead, we will read the refs manually. The
+             * documentation for this follows from https://git-scm.com/docs/gitrepository-layout and https://git-scm.com/docs/git-worktree.
+             *
+             * There are two cases to consider:
+             *  - a plain repository with .git directory at the root of the working tree
+             *  - a worktree with a plain text .git file at the root of the working tree
+             *
+             * In each case, our goal is to parse the HEAD file to get either a ref or a bare revision (in the case of being in detached
+             * HEAD state).
+             *
+             * In the case of a plain repository, we can read the HEAD file directly, resolved directly from the .git directory.
+             *
+             * In the case of a worktree, we read the gitdir from the plain text .git file. This resolves to a directory from which we read
+             * the HEAD file and resolve commondir to the plain git repository.
+             */
+            final Path dotGit = project.getRootProject().getRootDir().toPath().resolve(".git");
+            final String revision;
+            if (Files.exists(dotGit) == false) {
+                return "unknown";
+            }
+            final Path head;
+            final Path gitDir;
+            if (Files.isDirectory(dotGit)) {
+                // this is a git repository, we can read HEAD directly
+                head = dotGit.resolve("HEAD");
+                gitDir = dotGit;
+            } else {
+                // this is a git worktree, follow the pointer to the repository
+                final Path workTree = Paths.get(readFirstLine(dotGit).substring("gitdir:".length()).trim());
+                head = workTree.resolve("HEAD");
+                final Path commonDir = Paths.get(readFirstLine(workTree.resolve("commondir")));
+                if (commonDir.isAbsolute()) {
+                    gitDir = commonDir;
+                } else {
+                    // this is the common case
+                    gitDir = workTree.resolve(commonDir);
                 }
-                return gitRevision.get();
             }
-        };
+            final String ref = readFirstLine(head);
+            if (ref.startsWith("ref:")) {
+                revision = readFirstLine(gitDir.resolve(ref.substring("ref:".length()).trim()));
+            } else {
+                // we are in detached HEAD state
+                revision = ref;
+            }
+            return revision;
+        } catch (final IOException e) {
+            // for now, do not be lenient until we have better understanding of real-world scenarios where this happens
+            throw new GradleException("unable to read the git revision", e);
+        }
+    }
 
+    private String readFirstLine(final Path path) throws IOException {
+        return Files.lines(path, StandardCharsets.UTF_8)
+            .findFirst()
+            .orElseThrow(() -> new IOException("file [" + path + "] is empty"));
     }
 
 }

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -21,8 +21,11 @@
 
 import org.gradle.api.DefaultTask;
 import org.gradle.api.file.Directory;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.TaskAction;
 
 import java.util.ArrayList;
@@ -31,36 +34,66 @@
 
 public class BatsTestTask extends DefaultTask {
 
-    private Directory testsDir;
-    private Directory utilsDir;
-    private Directory archivesDir;
+    private final DirectoryProperty testsDir;
+    private final DirectoryProperty utilsDir;
+    private final DirectoryProperty distributionsDir;
+    private final DirectoryProperty pluginsDir;
+    private final DirectoryProperty upgradeDir;
     private String packageName;
 
+    public BatsTestTask() {
+        this.testsDir = getProject().getObjects().directoryProperty();
+        this.utilsDir = getProject().getObjects().directoryProperty();
+        this.distributionsDir = getProject().getObjects().directoryProperty();
+        this.pluginsDir = getProject().getObjects().directoryProperty();
+        this.upgradeDir = getProject().getObjects().directoryProperty();
+    }
+
     @InputDirectory
-    public Directory getTestsDir() {
+    public Provider<Directory> getTestsDir() {
         return testsDir;
     }
 
     public void setTestsDir(Directory testsDir) {
-        this.testsDir = testsDir;
+        this.testsDir.set(testsDir);
     }
 
     @InputDirectory
-    public Directory getUtilsDir() {
+    public Provider<Directory> getUtilsDir() {
         return utilsDir;
     }
 
     public void setUtilsDir(Directory utilsDir) {
-        this.utilsDir = utilsDir;
+        this.utilsDir.set(utilsDir);
+    }
+
+    @InputDirectory
+    public Provider<Directory> getDistributionsDir() {
+        return distributionsDir;
+    }
+
+    public void setDistributionsDir(Provider<Directory> distributionsDir) {
+        this.distributionsDir.set(distributionsDir);
+    }
+
+    @InputDirectory
+    @Optional
+    public Provider<Directory> getPluginsDir() {
+        return pluginsDir;
+    }
+
+    public void setPluginsDir(Provider<Directory> pluginsDir) {
+        this.pluginsDir.set(pluginsDir);
     }
 
     @InputDirectory
-    public Directory getArchivesDir() {
-        return archivesDir;
+    @Optional
+    public Provider<Directory> getUpgradeDir() {
+        return upgradeDir;
     }
 
-    public void setArchivesDir(Directory archivesDir) {
-        this.archivesDir = archivesDir;
+    public void setUpgradeDir(Provider<Directory> upgradeDir) {
+        this.upgradeDir.set(upgradeDir);
     }
 
     @Input
@@ -81,10 +114,16 @@ public void runBats() {
             .filter(f -> f.getName().endsWith(".bats"))
             .sorted().collect(Collectors.toList()));
         getProject().exec(spec -> {
-            spec.setWorkingDir(archivesDir.getAsFile());
+            spec.setWorkingDir(distributionsDir.getAsFile());
             spec.environment(System.getenv());
-            spec.environment("BATS_TESTS", testsDir.getAsFile().toString());
-            spec.environment("BATS_UTILS", utilsDir.getAsFile().toString());
+            spec.environment("BATS_TESTS", testsDir.getAsFile().get().toString());
+            spec.environment("BATS_UTILS", utilsDir.getAsFile().get().toString());
+            if (pluginsDir.isPresent()) {
+                spec.environment("BATS_PLUGINS", pluginsDir.getAsFile().get().toString());
+            }
+            if (upgradeDir.isPresent()) {
+                spec.environment("BATS_UPGRADE", upgradeDir.getAsFile().get().toString());
+            }
             spec.environment("PACKAGE_NAME", packageName);
             spec.setCommandLine(command);
         });

buildSrc/src/main/java/org/elasticsearch/gradle/test/BatsTestTask.java

@@ -7,7 +7,7 @@
   <property name="charset" value="UTF-8" />
 
   <module name="SuppressionFilter">
-    <property name="file" value="${suppressions}" />
+    <property name="file" value="${config_loc}/checkstyle_suppressions.xml" />
   </module>
 
   <!-- Checks Java files and forbids empty Javadoc comments -->