[Backport] [2.x] Update Gradle to 8.0 (#5666)

build.gradle

@@ -45,6 +45,7 @@ import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.EclipseJdt
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.api.Project;
+import org.gradle.api.internal.tasks.testing.junit.JUnitTestFramework
 import org.gradle.process.ExecResult;
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
@@ -55,8 +56,8 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.4.2" apply false
-  id "org.gradle.test-retry" version "1.4.1" apply false
+  id "com.diffplug.spotless" version "6.11.0" apply false
+  id "org.gradle.test-retry" version "1.5.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }
@@ -77,6 +78,19 @@ allprojects {
   group = 'org.opensearch'
   version = VersionProperties.getOpenSearch()
   description = "OpenSearch subproject ${project.path}"
+
+  afterEvaluate {
+    project.tasks.withType(Test) { task ->
+      // This is so hacky: now, by default, test tasks uses JUnit framework and always includes 'junit'
+      // JARs from the Gradle distribution (no ways to override this behavior). It causes JAR hell on test 
+      // classpath, example of the report:
+      //
+      //  jar1: /home/ubuntu/.gradle/caches/modules-2/files-2.1/junit/junit/4.13.2/8ac9e16d933b6fb43bc7f576336b8f4d7eb5ba12/junit-4.13.2.jar
+      //  jar2: /home/ubuntu/.gradle/wrapper/dists/gradle-8.0-rc-1-all/2p8rgxxewg8l61n1p3vrzr9s8/gradle-8.0-rc-1/lib/junit-4.13.2.jar
+      // 
+      task.getTestFrameworkProperty().convention(getProviderFactory().provider(() -> new JUnitTestFramework(task, task.getFilter(), false)));
+    }
+  }
 }
 
 configure(allprojects - project(':distribution:archives:integ-test-zip')) {

buildSrc/build.gradle

@@ -105,7 +105,7 @@ dependencies {
   api 'commons-codec:commons-codec:1.15'
   api 'org.apache.commons:commons-compress:1.22'
   api 'org.apache.ant:ant:1.10.13'
-  api 'com.netflix.nebula:gradle-extra-configurations-plugin:8.0.0'
+  api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:19.2.0'
   api 'com.netflix.nebula:gradle-info-plugin:12.0.0'
   api 'org.apache.rat:apache-rat:0.15'

buildSrc/src/main/java/org/opensearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -39,6 +39,7 @@
 import org.gradle.api.Task;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.services.internal.BuildServiceRegistryInternal;
+import org.gradle.api.services.internal.BuildServiceProvider;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Nested;
@@ -123,7 +124,7 @@ public List<ResourceLock> getSharedResources() {
             serviceRegistry,
             TestClustersPlugin.THROTTLE_SERVICE_NAME
         );
-        SharedResource resource = serviceRegistry.forService(throttleProvider);
+        final SharedResource resource = getSharedResource(serviceRegistry, throttleProvider);
 
         int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
         if (nodeCount > 0) {
@@ -133,6 +134,37 @@ public List<ResourceLock> getSharedResources() {
         return Collections.unmodifiableList(locks);
     }
 
+    private SharedResource getSharedResource(
+        BuildServiceRegistryInternal serviceRegistry,
+        Provider<TestClustersThrottle> throttleProvider
+    ) {
+        try {
+            try {
+                // The forService(Provider) is used by Gradle pre-8.0
+                return (SharedResource) MethodHandles.publicLookup()
+                    .findVirtual(
+                        BuildServiceRegistryInternal.class,
+                        "forService",
+                        MethodType.methodType(SharedResource.class, Provider.class)
+                    )
+                    .bindTo(serviceRegistry)
+                    .invokeExact(throttleProvider);
+            } catch (NoSuchMethodException | IllegalAccessException ex) {
+                // The forService(BuildServiceProvider) is used by Gradle post-8.0
+                return (SharedResource) MethodHandles.publicLookup()
+                    .findVirtual(
+                        BuildServiceRegistryInternal.class,
+                        "forService",
+                        MethodType.methodType(SharedResource.class, BuildServiceProvider.class)
+                    )
+                    .bindTo(serviceRegistry)
+                    .invokeExact((BuildServiceProvider<TestClustersThrottle, ?>) throttleProvider);
+            }
+        } catch (Throwable ex) {
+            throw new IllegalStateException("Unable to find suitable BuildServiceRegistryInternal::forService", ex);
+        }
+    }
+
     private ResourceLock getResourceLock(SharedResource resource, int nUsages) {
         try {
             try {

buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java

@@ -33,6 +33,9 @@
 package org.opensearch.gradle;
 
 import org.opensearch.gradle.test.GradleUnitTestCase;
+
+import java.util.UUID;
+
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.testfixtures.ProjectBuilder;
@@ -148,7 +151,8 @@ private void createJdk(Project project, String name, String vendor, String versi
     }
 
     private Project createProject() {
-        Project project = ProjectBuilder.builder().withParent(rootProject).build();
+        final String name = UUID.randomUUID().toString();
+        Project project = ProjectBuilder.builder().withName(name).withParent(rootProject).build();
         project.getPlugins().apply("opensearch.jdk-download");
         return project;
     }

buildSrc/src/test/java/org/opensearch/gradle/precommit/DependencyLicensesTaskTests.java

@@ -35,6 +35,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
@@ -95,7 +96,7 @@ public void givenProjectWithoutLicensesDirButWithDependenciesThenShouldThrowExce
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("does not exist, but there are dependencies"));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
         task.get().checkDependencies();
     }
 
@@ -113,7 +114,7 @@ public void givenProjectWithDependencyButNoShaFileThenShouldReturnException() th
         createFileIn(licensesDir, "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT);
         createFileIn(licensesDir, "groovy-NOTICE.txt", "");
 
-        project.getDependencies().add("compileClasspath", project.getDependencies().localGroovy());
+        project.getDependencies().add("someCompileConfiguration", project.getDependencies().localGroovy());
         task.get().checkDependencies();
     }
 
@@ -122,7 +123,7 @@ public void givenProjectWithDependencyButNoLicenseFileThenShouldReturnException(
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Missing LICENSE for "));
 
-        project.getDependencies().add("compileClasspath", project.getDependencies().localGroovy());
+        project.getDependencies().add("someCompileConfiguration", project.getDependencies().localGroovy());
 
         getLicensesDir(project).mkdir();
         updateShas.updateShas();
@@ -134,7 +135,7 @@ public void givenProjectWithDependencyButNoNoticeFileThenShouldReturnException()
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Missing NOTICE for "));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT);
 
@@ -147,7 +148,7 @@ public void givenProjectWithStrictDependencyButNoSourcesFileThenShouldReturnExce
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Missing SOURCES for "));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", STRICT_LICENSE_TEXT);
         createFileIn(getLicensesDir(project), "groovy-NOTICE.txt", "");
@@ -158,7 +159,7 @@ public void givenProjectWithStrictDependencyButNoSourcesFileThenShouldReturnExce
 
     @Test
     public void givenProjectWithStrictDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", STRICT_LICENSE_TEXT);
         createFileIn(getLicensesDir(project), "groovy-NOTICE.txt", "");
@@ -174,7 +175,7 @@ public void givenProjectWithStrictDependencyAndEverythingInOrderThenShouldReturn
 
     @Test
     public void givenProjectWithDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
 
@@ -187,7 +188,7 @@ public void givenProjectWithALicenseButWithoutTheDependencyThenShouldThrowExcept
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Unused license "));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createAllDefaultDependencyFiles(licensesDir, "groovy", "javaparser-core");
@@ -201,7 +202,7 @@ public void givenProjectWithANoticeButWithoutTheDependencyThenShouldThrowExcepti
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Unused notice "));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createAllDefaultDependencyFiles(licensesDir, "groovy", "javaparser-core");
@@ -215,7 +216,7 @@ public void givenProjectWithAShaButWithoutTheDependencyThenShouldThrowException(
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("Unused sha files found: \n"));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createAllDefaultDependencyFiles(licensesDir, "groovy", "javaparser-core");
@@ -229,7 +230,7 @@ public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() th
         expectedException.expect(GradleException.class);
         expectedException.expectMessage(containsString("SHA has changed! Expected "));
 
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createAllDefaultDependencyFiles(licensesDir, "groovy");
@@ -243,7 +244,7 @@ public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() th
 
     @Test
     public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws Exception {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createAllDefaultDependencyFiles(licensesDir, "groovy", "javaparser");
@@ -258,7 +259,7 @@ public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws
 
     @Test
     public void givenProjectWithAIgnoreShaConfigurationAndNoShaFileThenShouldReturnSilently() throws Exception {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File licensesDir = getLicensesDir(project);
         createFileIn(licensesDir, "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT);
@@ -297,6 +298,11 @@ private Project createProject() {
         Project project = ProjectBuilder.builder().build();
         project.getPlugins().apply(JavaPlugin.class);
 
+        Configuration compileClasspath = project.getConfigurations().getByName("compileClasspath");
+        Configuration someCompileConfiguration = project.getConfigurations().create("someCompileConfiguration");
+        // Declare a configuration that is going to resolve the compile classpath of the application
+        project.getConfigurations().add(compileClasspath.extendsFrom(someCompileConfiguration));
+
         return project;
     }
 

buildSrc/src/test/java/org/opensearch/gradle/precommit/UpdateShasTaskTests.java

@@ -36,6 +36,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
@@ -86,7 +87,7 @@ public void whenDependencyDoesntExistThenShouldDeleteDependencySha() throws IOEx
 
     @Test
     public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile() throws IOException, NoSuchAlgorithmException {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         getLicensesDir(project).mkdir();
         task.updateShas();
@@ -99,7 +100,7 @@ public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile() throws IOE
 
     @Test
     public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile() throws IOException, NoSuchAlgorithmException {
-        project.getDependencies().add("compileClasspath", dependency);
+        project.getDependencies().add("someCompileConfiguration", dependency);
 
         File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next();
         String groovyShaName = groovyJar.getName() + ".sha1";
@@ -122,6 +123,11 @@ private Project createProject() {
         Project project = ProjectBuilder.builder().build();
         project.getPlugins().apply(JavaPlugin.class);
 
+        Configuration compileClasspath = project.getConfigurations().getByName("compileClasspath");
+        Configuration someCompileConfiguration = project.getConfigurations().create("someCompileConfiguration");
+        // Declare a configuration that is going to resolve the compile classpath of the application
+        project.getConfigurations().add(compileClasspath.extendsFrom(someCompileConfiguration));
+
         return project;
     }
 

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -41,7 +41,7 @@ dependencies {
 ["0.0.1"].forEach { v ->
   ["opensearch", "other"].forEach { p ->
     tasks.register("broken-log4j-${p}-${v}", Jar) {
-      destinationDir = file("${buildDir}/testrepo/org/${p}/gradle/broken-log4j/${v}/")
+      destinationDirectory = file("${buildDir}/testrepo/org/${p}/gradle/broken-log4j/${v}/")
       archiveFileName = "broken-log4j-${v}.jar"
       from sourceSets.main.output
       include "**/TestingLog4j.class"
@@ -51,7 +51,7 @@ dependencies {
 }
 
 tasks.register("jarhellJdk", Jar) {
-  destinationDir = file("${buildDir}/testrepo/org/other/gradle/jarhellJdk/0.0.1/")
+  destinationDirectory = file("${buildDir}/testrepo/org/other/gradle/jarhellJdk/0.0.1/")
   archiveFileName = "jarhellJdk-0.0.1.jar"
   from sourceSets.main.output
   include "**/String.class"

buildSrc/version.properties

@@ -9,7 +9,7 @@ spatial4j         = 0.7
 jts               = 1.15.0
 jackson           = 2.14.2
 jackson_databind  = 2.14.2
-snakeyaml         = 1.32
+snakeyaml         = 1.33
 icu4j             = 70.1
 supercsv          = 2.4.0
 # Update to 2.17.2+ is breaking OpenSearchJsonLayout (see https://issues.apache.org/jira/browse/LOG4J2-3562)
@@ -20,7 +20,8 @@ jettison          = 1.5.1
 woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1
-
+guava             = 31.1-jre
+
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "nebula.ospackage-base" version "9.1.1"
+  id "com.netflix.nebula.ospackage-base" version "11.0.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {
@@ -290,7 +290,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
   }
 }
 
-apply plugin: 'nebula.ospackage-base'
+apply plugin: 'com.netflix.nebula.ospackage-base'
 
 // this is package indepdendent configuration
 ospackage {

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=312eb12875e1747e05c2f81a4789902d7e4ec5defbd1eefeaccc08acf096505d
+distributionSha256Sum=f30b29580fe11719087d698da23f3b0f0d04031d8995f7dd8275a31f7674dc01