From 95735e9406bf6fc7d746cc1fa9435d61a081c21f Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 21 Nov 2024 14:45:59 +0100 Subject: [PATCH 1/8] Run checkpart 4 with configuration cache enabled --- .buildkite/pipelines/pull-request/part-4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index af11f08953d07..120fe948f92e6 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -2,7 +2,7 @@ config: skip-target-branches: "7.17" steps: - label: part-4 - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4 + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4 --configuration-cache timeout_in_minutes: 300 agents: provider: gcp From afc48b0ee2630c437a679463e69ffe13044026a1 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 21 Nov 2024 15:31:25 +0100 Subject: [PATCH 2/8] We need to postpone resolving minimum java version from bwc branch to task execution phase --- .../gradle/internal/BwcSetupExtension.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index d7bf839817e12..f487381dcd068 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -16,6 +16,7 @@ import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; +import org.gradle.api.Task; import org.gradle.api.logging.LogLevel; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; @@ -103,10 +104,17 @@ private static TaskProvider createRunBwcGradleTask( loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.getWorkingDir().set(checkoutDir.get()); - loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> { - spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); - spec.getParameters().getCheckoutDir().set(checkoutDir); - }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); + + loggedExec.doFirst(new Action() { + @Override + public void execute(Task task) { + Provider minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> { + spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); + spec.getParameters().getCheckoutDir().set(checkoutDir); + }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))); + loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get()); + } + }); if (isCi && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted @@ -169,15 +177,8 @@ private static Provider getJavaHome(ObjectFactory objectFactory, JavaToo .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } - private static String readFromFile(File file) { - try { - return FileUtils.readFileToString(file).trim(); - } catch (IOException ioException) { - throw new GradleException("Cannot read java properties file.", ioException); - } - } - public static abstract class JavaHomeValueSource implements ValueSource { + public abstract static class JavaHomeValueSource implements ValueSource { private String minimumCompilerVersionPath(Version bwcVersion) { return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION)) @@ -185,6 +186,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) { : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; } + private static String readFromFile(File file) { + try { + return FileUtils.readFileToString(file).trim(); + } catch (IOException ioException) { + throw new GradleException("Cannot read java properties file.", ioException); + } + } + @Override public String obtain() { return readFromFile( From 933e06794a8fce60547e045948e18989edec89f6 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 21 Nov 2024 17:01:02 +0100 Subject: [PATCH 3/8] fix spotless --- .../org/elasticsearch/gradle/internal/BwcSetupExtension.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index f487381dcd068..15fa705c8d3e1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -104,7 +104,6 @@ private static TaskProvider createRunBwcGradleTask( loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.getWorkingDir().set(checkoutDir.get()); - loggedExec.doFirst(new Action() { @Override public void execute(Task task) { @@ -177,8 +176,7 @@ private static Provider getJavaHome(ObjectFactory objectFactory, JavaToo .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } - - public abstract static class JavaHomeValueSource implements ValueSource { + public abstract static class JavaHomeValueSource implements ValueSource { private String minimumCompilerVersionPath(Version bwcVersion) { return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION)) From 004d5af80a598219223d62ec1984fd377fb30757 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 22 Nov 2024 15:44:07 +0100 Subject: [PATCH 4/8] Do not run checkPart4 on RAM --- .buildkite/pipelines/pull-request/part-4.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index 120fe948f92e6..a3e858f93446a 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -8,4 +8,3 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 - buildDirectory: /dev/shm/bk From 3909f9d73480caa5f0a736e92eefa39b97fb49bd Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 22 Nov 2024 18:10:13 +0100 Subject: [PATCH 5/8] Fix tests with cc in x-pack/plugin/sql/qa/jdbc/security --- x-pack/plugin/sql/qa/jdbc/security/build.gradle | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle index 82510285cb996..6ef1a133a2bc7 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle @@ -58,15 +58,16 @@ subprojects { project.gradle.sharedServices, TestClustersPlugin.REGISTRY_SERVICE_NAME ) - project.getProviders().of(TestClusterValueSource.class) { + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { it.parameters.path.set(clusterPath) it.parameters.clusterName.set("javaRestTest") it.parameters.service = serviceProvider } nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + "${-> clusterInfo.map { it.auditLogs.get(0) } }" nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}.json" + "${-> clusterInfo.map { it.auditLogs.get(0).getParentFile()} }/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}.json" } } From 5cf96be6b27067cef38cdbe12ec3c2b074a7b300 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 22 Nov 2024 20:44:55 +0100 Subject: [PATCH 6/8] Fix more tests for cc in x-pack/plugin/sql/qa --- .../plugin/sql/qa/jdbc/security/build.gradle | 8 +++--- .../sql/qa/server/security/build.gradle | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle index 6ef1a133a2bc7..b40a8aaaa5580 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle @@ -50,6 +50,7 @@ subprojects { tasks.withType(RestIntegTestTask).configureEach { + def taskName = name dependsOn copyTestClasses classpath += configurations.testArtifacts testClassesDirs = project.files(testArtifactsDir) @@ -61,13 +62,12 @@ subprojects { def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("javaRestTest") + it.parameters.clusterName.set(taskName) it.parameters.service = serviceProvider } - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> clusterInfo.map { it.auditLogs.get(0) } }" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> clusterInfo.map { it.auditLogs.get(0).getParentFile()} }/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}.json" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json" } } } diff --git a/x-pack/plugin/sql/qa/server/security/build.gradle b/x-pack/plugin/sql/qa/server/security/build.gradle index 2d9f7b563d073..ef81ebda5cb89 100644 --- a/x-pack/plugin/sql/qa/server/security/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/build.gradle @@ -1,3 +1,9 @@ +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.testclusters.TestClusterValueSource +import org.elasticsearch.gradle.testclusters.TestClustersPlugin +import org.elasticsearch.gradle.testclusters.TestClustersRegistry +import org.elasticsearch.gradle.util.GradleUtils + apply plugin: 'elasticsearch.internal-test-artifact' dependencies { @@ -16,6 +22,8 @@ subprojects { // Use tests from the root security qa project in subprojects configurations.create('testArtifacts').transitive(false) + def clusterPath = getPath() + dependencies { javaRestTestImplementation project(":x-pack:plugin:core") javaRestTestImplementation(testArtifact(project(xpackModule('core')))) @@ -50,12 +58,23 @@ subprojects { tasks.named("javaRestTest").configure { dependsOn copyTestClasses + + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + TestClustersPlugin.REGISTRY_SERVICE_NAME + ) + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set("javaRestTest") + it.parameters.service = serviceProvider + } + testClassesDirs += project.files(testArtifactsDir) classpath += configurations.testArtifacts - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz"} } } From 7a16c346beec81409880c3914b42d4c1161a9d83 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 22 Nov 2024 23:21:39 +0100 Subject: [PATCH 7/8] Test with beefy build agent --- .buildkite/pipelines/pull-request/part-4.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index a3e858f93446a..c3367a37f31be 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -2,9 +2,10 @@ config: skip-target-branches: "7.17" steps: - label: part-4 - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4 --configuration-cache + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4 --configuration-cache --stacktrace timeout_in_minutes: 300 agents: provider: gcp image: family/elasticsearch-ubuntu-2004 - machineType: custom-32-98304 + machineType: n2-highmem-32 + buildDirectory: /dev/shm/bk From a1149d6b855b7d504c5fdcc830a72ed0518ae6f4 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Sat, 23 Nov 2024 00:58:28 +0100 Subject: [PATCH 8/8] Try again with given machine type --- .buildkite/pipelines/pull-request/part-4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index c3367a37f31be..a19d61e7792e7 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -7,5 +7,5 @@ steps: agents: provider: gcp image: family/elasticsearch-ubuntu-2004 - machineType: n2-highmem-32 + machineType: custom-32-98304 buildDirectory: /dev/shm/bk