From db21521fcfea08580b6fb3ee1e3c30dfe4ea3ec5 Mon Sep 17 00:00:00 2001 From: "xieran.sai" Date: Mon, 5 Jun 2023 00:26:13 +0800 Subject: [PATCH 1/2] turbine support worker mode --- .../google/devtools/build/java/turbine/BUILD | 8 ++- .../java/turbine/TurbineWorkerWrapper.java | 51 +++++++++++++++++++ .../rules/java/JavaHeaderCompileAction.java | 9 +++- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/BUILD b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/BUILD index d09d4caf2d4022..8a4350e8ff8bb7 100644 --- a/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/BUILD +++ b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/BUILD @@ -7,12 +7,16 @@ package( java_binary( name = "turbine_direct_binary", - main_class = "com.google.turbine.main.Main", + main_class = "com.google.devtools.build.java.turbine.TurbineWorkerWrapper", + srcs = ["TurbineWorkerWrapper.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/worker:work_request_handlers", + "//third_party:turbine", + ], runtime_deps = [ "//src/main/protobuf:deps_java_proto", "//third_party:guava", "//third_party:jsr305", - "//third_party:turbine", ], ) diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java new file mode 100644 index 00000000000000..f593759b938165 --- /dev/null +++ b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java @@ -0,0 +1,51 @@ +package com.google.devtools.build.java.turbine; + +import com.google.turbine.main.Main; +import com.google.devtools.build.lib.worker.ProtoWorkerMessageProcessor; +import com.google.devtools.build.lib.worker.WorkRequestHandler; + +import java.io.IOException; +import java.io.PrintWriter; +import java.time.Duration; +import java.util.List; + +/** + * A Wrapper for Turbine to support multiplex worker + */ +public class TurbineWorkerWrapper { + + public static void main(String[] args) throws IOException { + if (args.length == 1 && args[0].equals("--persistent_worker")) { + WorkRequestHandler workerHandler = + new WorkRequestHandler.WorkRequestHandlerBuilder( + TurbineWorkerWrapper::turbine, + System.err, + new ProtoWorkerMessageProcessor(System.in, System.out)) + .setCpuUsageBeforeGc(Duration.ofSeconds(10)) + .build(); + int exitCode = 1; + try { + workerHandler.processRequests(); + exitCode = 0; + } catch (IOException e) { + System.err.println(e.getMessage()); + } finally { + // Prevent hanging threads from keeping the worker alive. + System.exit(exitCode); + } + } else { + Main.main(args); + } + } + + private static int turbine(List args, PrintWriter pw) { + try { + Main.compile(args.toArray(new String[0])); + } catch (Throwable e) { + System.err.println(e.getMessage()); + return 1; + } + return 0; + } + +} \ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java index 5bf8e6274aa9c0..9fcb10df614d79 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java @@ -132,7 +132,7 @@ public static Builder newBuilder(RuleContext ruleContext) { public static final class Builder { private static final ParamFileInfo PARAM_FILE_INFO = - ParamFileInfo.builder(UNQUOTED).setCharset(ISO_8859_1).build(); + ParamFileInfo.builder(UNQUOTED).setCharset(ISO_8859_1).setUseAlways(true).build(); private final RuleContext ruleContext; @@ -457,6 +457,11 @@ public void build(JavaToolchainProvider javaToolchain) { ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, outputDepsProto.getExecPathString()); } + executionInfo = ImmutableMap.builder() + .putAll(executionInfo) + .putAll(ExecutionRequirements.WORKER_MODE_ENABLED) + .putAll(ExecutionRequirements.WORKER_MULTIPLEX_MODE_ENABLED) + .build(); if (useDirectClasspath) { NestedSet classpath; if (!directJars.isEmpty() || classpathEntries.isEmpty()) { @@ -485,7 +490,7 @@ public void build(JavaToolchainProvider javaToolchain) { ruleContext.registerAction( new JavaHeaderCompileAction( /* owner= */ ruleContext.getActionOwner(), - /* tools= */ NestedSetBuilder.emptySet(Order.STABLE_ORDER), + /* tools= */ headerCompiler.tool().getFilesToRun(), /* inputs= */ allInputs, /* outputs= */ outputs.build(), /* resourceSetOrBuilder= */ AbstractAction.DEFAULT_RESOURCE_SET, From a1b3c1deb27b082a10402b9c053fda5568cc326b Mon Sep 17 00:00:00 2001 From: "xieran.sai" Date: Wed, 5 Jul 2023 14:48:39 +0800 Subject: [PATCH 2/2] add flag --- .../java/turbine/TurbineWorkerWrapper.java | 12 ++++++---- .../rules/java/JavaHeaderCompileAction.java | 24 ++++++++++--------- .../build/lib/rules/java/JavaToolchain.java | 6 +++++ .../lib/rules/java/JavaToolchainProvider.java | 20 ++++++++++++++++ .../lib/rules/java/JavaToolchainRule.java | 8 +++++++ 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java index f593759b938165..3db5dfce3d8a1e 100644 --- a/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java +++ b/src/java_tools/buildjar/java/com/google/devtools/build/java/turbine/TurbineWorkerWrapper.java @@ -5,6 +5,7 @@ import com.google.devtools.build.lib.worker.WorkRequestHandler; import java.io.IOException; +import java.io.PrintStream; import java.io.PrintWriter; import java.time.Duration; import java.util.List; @@ -16,10 +17,13 @@ public class TurbineWorkerWrapper { public static void main(String[] args) throws IOException { if (args.length == 1 && args[0].equals("--persistent_worker")) { + PrintStream realStdErr = System.err; WorkRequestHandler workerHandler = new WorkRequestHandler.WorkRequestHandlerBuilder( - TurbineWorkerWrapper::turbine, - System.err, + new WorkRequestHandler.WorkRequestCallback( + (request, pw) -> + turbine(request.getArgumentsList(), pw)), + realStdErr, new ProtoWorkerMessageProcessor(System.in, System.out)) .setCpuUsageBeforeGc(Duration.ofSeconds(10)) .build(); @@ -28,7 +32,7 @@ public static void main(String[] args) throws IOException { workerHandler.processRequests(); exitCode = 0; } catch (IOException e) { - System.err.println(e.getMessage()); + realStdErr.println(e.getMessage()); } finally { // Prevent hanging threads from keeping the worker alive. System.exit(exitCode); @@ -42,7 +46,7 @@ private static int turbine(List args, PrintWriter pw) { try { Main.compile(args.toArray(new String[0])); } catch (Throwable e) { - System.err.println(e.getMessage()); + pw.println(e.getMessage()); return 1; } return 0; diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java index 9fcb10df614d79..122740dd91b3a7 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHeaderCompileAction.java @@ -449,19 +449,21 @@ public void build(JavaToolchainProvider javaToolchain) { } } - ImmutableMap executionInfo = - TargetUtils.getExecutionInfo(ruleContext.getRule(), ruleContext.isAllowTagsPropagation()); + ImmutableMap.Builder executionInfoBuilder = ImmutableMap.builder(); + executionInfoBuilder.putAll( + TargetUtils.getExecutionInfo(ruleContext.getRule(), ruleContext.isAllowTagsPropagation())); if (javaConfiguration.inmemoryJdepsFiles()) { - executionInfo = - ImmutableMap.of( - ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, - outputDepsProto.getExecPathString()); + executionInfoBuilder.put( + ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS, + outputDepsProto.getExecPathString()); } - executionInfo = ImmutableMap.builder() - .putAll(executionInfo) - .putAll(ExecutionRequirements.WORKER_MODE_ENABLED) - .putAll(ExecutionRequirements.WORKER_MULTIPLEX_MODE_ENABLED) - .build(); + if (javaToolchain.getHeaderCompilerSupportsWorkers()) { + executionInfoBuilder.putAll(ExecutionRequirements.WORKER_MODE_ENABLED); + } + if (javaToolchain.getHeaderCompilerSupportsMultiplexWorkers()) { + executionInfoBuilder.putAll(ExecutionRequirements.WORKER_MULTIPLEX_MODE_ENABLED); + } + ImmutableMap executionInfo = executionInfoBuilder.build(); if (useDirectClasspath) { NestedSet classpath; if (!directJars.isEmpty() || classpathEntries.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java index f1f4e41187ee39..e355fbb5c0abe8 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java @@ -68,6 +68,10 @@ public ConfiguredTarget create(RuleContext ruleContext) ruleContext.attributes().get("javac_supports_multiplex_workers", Type.BOOLEAN); boolean javacSupportsWorkerCancellation = ruleContext.attributes().get("javac_supports_worker_cancellation", Type.BOOLEAN); + boolean headerCompilerSupportsWorkers = + ruleContext.attributes().get("header_compiler_supports_workers", Type.BOOLEAN); + boolean headerCompilerSupportsMultiplexWorkers = + ruleContext.attributes().get("header_compiler_supports_multiplex_workers", Type.BOOLEAN); ImmutableSet headerCompilerBuiltinProcessors = ImmutableSet.copyOf( ruleContext.attributes().get("header_compiler_builtin_processors", Type.STRING_LIST)); @@ -168,6 +172,8 @@ public ConfiguredTarget create(RuleContext ruleContext) javacSupportsWorkers, javacSupportsMultiplexWorkers, javacSupportsWorkerCancellation, + headerCompilerSupportsWorkers, + headerCompilerSupportsMultiplexWorkers, bootclasspath, tools, javabuilder, diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java index 870fc1c2340546..9598b4aaea2e6f 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java @@ -97,6 +97,8 @@ public static JavaToolchainProvider create( boolean javacSupportsWorkers, boolean javacSupportsMultiplexWorkers, boolean javacSupportsWorkerCancellation, + boolean headerCompilerSupportsWorkers, + boolean headerCompilerSupportsMultiplexWorkers, BootClassPathInfo bootclasspath, NestedSet tools, JavaToolchainTool javaBuilder, @@ -149,6 +151,8 @@ public static JavaToolchainProvider create( javacSupportsWorkers, javacSupportsMultiplexWorkers, javacSupportsWorkerCancellation, + headerCompilerSupportsWorkers, + headerCompilerSupportsMultiplexWorkers, packageConfiguration, jacocoRunner, proguardAllowlister, @@ -182,6 +186,8 @@ public static JavaToolchainProvider create( private final boolean javacSupportsWorkers; private final boolean javacSupportsMultiplexWorkers; private final boolean javacSupportsWorkerCancellation; + private final boolean headerCompilerSupportsWorkers; + private final boolean headerCompilerSupportsMultiplexWorkers; private final ImmutableList packageConfiguration; private final FilesToRunProvider jacocoRunner; private final FilesToRunProvider proguardAllowlister; @@ -215,6 +221,8 @@ private JavaToolchainProvider( boolean javacSupportsWorkers, boolean javacSupportsMultiplexWorkers, boolean javacSupportsWorkerCancellation, + boolean headerCompilerSupportsWorkers, + boolean headerCompilerSupportsMultiplexWorkers, ImmutableList packageConfiguration, FilesToRunProvider jacocoRunner, FilesToRunProvider proguardAllowlister, @@ -247,6 +255,8 @@ private JavaToolchainProvider( this.javacSupportsWorkers = javacSupportsWorkers; this.javacSupportsMultiplexWorkers = javacSupportsMultiplexWorkers; this.javacSupportsWorkerCancellation = javacSupportsWorkerCancellation; + this.headerCompilerSupportsWorkers = headerCompilerSupportsWorkers; + this.headerCompilerSupportsMultiplexWorkers = headerCompilerSupportsMultiplexWorkers; this.packageConfiguration = packageConfiguration; this.jacocoRunner = jacocoRunner; this.proguardAllowlister = proguardAllowlister; @@ -419,6 +429,16 @@ public boolean getJavacSupportsMultiplexWorkers() { return javacSupportsMultiplexWorkers; } + /** Returns whether JavaHeaderCompiler supports running as a persistent worker or not. */ + public boolean getHeaderCompilerSupportsWorkers() { + return headerCompilerSupportsWorkers; + } + + /** Returns whether JavaHeaderCompiler supports running persistent workers in multiplex mode */ + public boolean getHeaderCompilerSupportsMultiplexWorkers() { + return headerCompilerSupportsMultiplexWorkers; + } + /** Returns whether JavaBuilders supports running persistent workers with cancellation */ public boolean getJavacSupportsWorkerCancellation() { return javacSupportsWorkerCancellation; diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java index 506165a1e108a4..2f04d0444b949f 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java @@ -129,6 +129,14 @@ The Java target version (e.g., '6' or '7'). It specifies for which Java runtime True if JavaBuilder supports cancellation of persistent workers, false if it doesn't. */ .add(attr("javac_supports_worker_cancellation", BOOLEAN).value(true)) + /* + True if JavaHeaderCompiler supports running as a persistent worker, false if it doesn't. + */ + .add(attr("header_compiler_supports_workers", BOOLEAN).value(false)) + /* + True if JavaHeaderCompiler supports running as a multiplex persistent worker, false if it doesn't. + */ + .add(attr("header_compiler_supports_multiplex_workers", BOOLEAN).value(false)) /* Labels of tools available for label-expansion in jvm_opts. */