[7.1.0] Introduce --local_resources flag (#21331)

Follow up for
#19839 (comment),
introduce new flag that will replace all the others.

Closes #20398.

Commit
ea75928

PiperOrigin-RevId: 596905096
Change-Id: Ic2c20dae40efa797953cde6934085f899acb9b52

Co-authored-by: Alessandro Patti <ale812@yahoo.it>

src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java

@@ -16,6 +16,7 @@
 
 import com.google.devtools.build.lib.util.CpuResourceConverter;
 import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
 import com.google.devtools.common.options.OptionEffectTag;
@@ -106,24 +107,29 @@ public class AnalysisOptions extends OptionsBase {
 
   @Option(
       name = "experimental_skyframe_cpu_heavy_skykeys_thread_pool_size",
-      defaultValue = "HOST_CPUS",
+      defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
       documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
       metadataTags = OptionMetadataTag.EXPERIMENTAL,
       effectTags = {
         OptionEffectTag.LOADING_AND_ANALYSIS,
         OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION
       },
       help =
-          "If set to a positive value (e.g. \"HOST_CPUS*1.5\"), Skyframe will run the"
-              + " loading/analysis phase with 2 separate thread pools: 1 with <value> threads"
-              + " (ideally close to HOST_CPUS) reserved for CPU-heavy SkyKeys, and 1 \"standard\""
+          "If set to a positive value (e.g. \""
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + "*1.5\"),"
+              + " Skyframe will run the loading/analysis phase with 2 separate thread pools:"
+              + " 1 with <value> threads (ideally close to "
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + ")"
+              + " reserved for CPU-heavy SkyKeys, and 1 \"standard\""
               + " thread pool (whose size is controlled by --loading_phase_threads) for the rest.",
       converter = CpuResourceConverter.class)
   public int cpuHeavySkyKeysThreadPoolSize;
 
   @Option(
       name = "experimental_oom_sensitive_skyfunctions_semaphore_size",
-      defaultValue = "HOST_CPUS",
+      defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
       documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
       metadataTags = OptionMetadataTag.EXPERIMENTAL,
       effectTags = {
@@ -133,7 +139,9 @@ public class AnalysisOptions extends OptionsBase {
       help =
           "Sets the size of the semaphore used to prevent SkyFunctions with large peak memory"
               + " requirement from OOM-ing blaze. A value of 0 indicates that no semaphore should"
-              + " be used. Example value: \"HOST_CPUS*0.5\".",
+              + " be used. Example value: \""
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + "*0.5\".",
       converter = CpuResourceConverter.class)
   public int oomSensitiveSkyFunctionsSemaphoreSize;
 }

src/main/java/com/google/devtools/build/lib/analysis/BUILD

@@ -473,6 +473,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util:cpu_resource_converter",
+        "//src/main/java/com/google/devtools/build/lib/util:resource_converter",
         "//src/main/java/com/google/devtools/common/options",
     ],
 )

src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java

@@ -16,7 +16,6 @@
 import com.github.benmanes.caffeine.cache.CaffeineSpec;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.OptionsUtils;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -399,13 +398,9 @@ public String getSymlinkPrefix(String productName) {
   public int skymeldAnalysisOverlapPercentage;
 
   /** Converter for filesystem value checker threads. */
-  public static class ThreadConverter extends ResourceConverter {
+  public static class ThreadConverter extends ResourceConverter.IntegerConverter {
     public ThreadConverter() {
-      super(
-          /* autoSupplier= */ () ->
-              (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
-          /* minValue= */ 1,
-          /* maxValue= */ Integer.MAX_VALUE);
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
     }
   }
 
@@ -448,27 +443,24 @@ public ThreadConverter() {
    * Converter for jobs: Takes keyword ({@value #FLAG_SYNTAX}). Values must be between 1 and
    * MAX_JOBS.
    */
-  public static class JobsConverter extends ResourceConverter {
+  public static class JobsConverter extends ResourceConverter.IntegerConverter {
     public JobsConverter() {
-      super(
-          () -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
-          1,
-          MAX_JOBS);
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ MAX_JOBS);
     }
 
     @Override
-    public int checkAndLimit(int value) throws OptionsParsingException {
-      if (value < minValue) {
+    public Integer checkAndLimit(Integer value) throws OptionsParsingException {
+      if (value.doubleValue() < minValue) {
         throw new OptionsParsingException(
             String.format("Value '(%d)' must be at least %d.", value, minValue));
       }
-      if (value > maxValue) {
+      if (value.doubleValue() > maxValue) {
         logger.atWarning().log(
             "Flag remoteWorker \"jobs\" ('%d') was set too high. "
                 + "This is a result of passing large values to --local_resources or --jobs. "
                 + "Using '%d' jobs",
             value, maxValue);
-        value = maxValue;
+        return maxValue;
       }
       return value;
     }

src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java

@@ -24,6 +24,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.eventbus.Subscribe;
 import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.actions.Action;
@@ -115,7 +116,6 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
 /**
@@ -996,11 +996,16 @@ public static void configureResourceManager(ResourceManager resourceMgr, BuildRe
     ImmutableMap<String, Double> cpuRam =
         ImmutableMap.of(
             ResourceSet.CPU,
+            // Replace with 1.0 * ResourceConverter.HOST_CPUS.get() after flag deprecation
             options.localCpuResources,
             ResourceSet.MEMORY,
+            // Replace with 0.67 * ResourceConverter.HOST_RAM.get() after flag deprecation
             options.localRamResources);
     ImmutableMap<String, Double> resources =
-        Stream.concat(options.localExtraResources.stream(), cpuRam.entrySet().stream())
+        Streams.concat(
+                options.localExtraResources.stream(),
+                cpuRam.entrySet().stream(),
+                options.localResources.stream())
             .collect(
                 ImmutableMap.toImmutableMap(
                     Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v2));

src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java

@@ -293,35 +293,54 @@ public boolean shouldMaterializeParamFiles() {
 
   @Option(
       name = "local_cpu_resources",
-      defaultValue = "HOST_CPUS",
+      defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
+      deprecationWarning =
+          "--local_cpu_resources is deprecated, please use --local_resources=cpu= instead.",
       documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
       effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
       help =
           "Explicitly set the total number of local CPU cores available to Bazel to spend on build"
-              + " actions executed locally. Takes an integer, or \"HOST_CPUS\", optionally followed"
-              + " by [-|*]<float> (eg. HOST_CPUS*.5 to use half the available CPU cores).By"
-              + " default, (\"HOST_CPUS\"), Bazel will query system configuration to estimate"
-              + " the number of CPU cores available.",
+              + " actions executed locally. Takes an integer, or \""
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + "\", optionally followed"
+              + " by [-|*]<float> (eg. "
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + "*.5"
+              + " to use half the available CPU cores). By default, (\""
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + "\"), Bazel will query system"
+              + " configuration to estimate the number of CPU cores available.",
       converter = CpuResourceConverter.class)
   public double localCpuResources;
 
   @Option(
       name = "local_ram_resources",
-      defaultValue = "HOST_RAM*.67",
+      defaultValue = ResourceConverter.HOST_RAM_KEYWORD + "*.67",
+      deprecationWarning =
+          "--local_ram_resources is deprecated, please use --local_resources=memory= instead.",
       documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
       effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
       help =
           "Explicitly set the total amount of local host RAM (in MB) available to Bazel to spend on"
-              + " build actions executed locally. Takes an integer, or \"HOST_RAM\", optionally"
-              + " followed by [-|*]<float> (eg. HOST_RAM*.5 to use half the available RAM). By"
-              + " default, (\"HOST_RAM*.67\"), Bazel will query system configuration to estimate"
-              + " the amount of RAM available and will use 67% of it.",
+              + " build actions executed locally. Takes an integer, or \""
+              + ResourceConverter.HOST_RAM_KEYWORD
+              + "\", optionally followed by [-|*]<float>"
+              + " (eg. "
+              + ResourceConverter.HOST_RAM_KEYWORD
+              + "*.5 to use half the available"
+              + " RAM). By default, (\""
+              + ResourceConverter.HOST_RAM_KEYWORD
+              + "*.67\"),"
+              + " Bazel will query system configuration to estimate the amount of RAM available"
+              + " and will use 67% of it.",
       converter = RamResourceConverter.class)
   public double localRamResources;
 
   @Option(
       name = "local_extra_resources",
       defaultValue = "null",
+      deprecationWarning =
+          "--local_extra_resources is deprecated, please use --local_resources instead.",
       documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
       effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
       allowMultiple = true,
@@ -336,6 +355,31 @@ public boolean shouldMaterializeParamFiles() {
       converter = Converters.StringToDoubleAssignmentConverter.class)
   public List<Map.Entry<String, Double>> localExtraResources;
 
+  @Option(
+      name = "local_resources",
+      defaultValue = "null",
+      documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+      effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
+      allowMultiple = true,
+      help =
+          "Set the number of resources available to Bazel. "
+              + "Takes in an assignment to a float or "
+              + ResourceConverter.HOST_RAM_KEYWORD
+              + "/"
+              + ResourceConverter.HOST_CPUS_KEYWORD
+              + ", optionally "
+              + "followed by [-|*]<float> (eg. memory="
+              + ResourceConverter.HOST_RAM_KEYWORD
+              + "*.5 to use half the available RAM). "
+              + "Can be used multiple times to specify multiple "
+              + "types of resources. Bazel will limit concurrently running actions "
+              + "based on the available resources and the resources required. "
+              + "Tests can declare the amount of resources they need "
+              + "by using a tag of the \"resources:<resource name>:<amount>\" format. "
+              + "Overrides resources specified by --local_{cpu|ram|extra}_resources.",
+      converter = ResourceConverter.AssignmentConverter.class)
+  public List<Map.Entry<String, Double>> localResources;
+
   @Option(
       name = "local_test_jobs",
       defaultValue = "auto",
@@ -574,9 +618,9 @@ public String getTypeDescription() {
   }
 
   /** Converter for --local_test_jobs, which takes {@value FLAG_SYNTAX} */
-  public static class LocalTestJobsConverter extends ResourceConverter {
+  public static class LocalTestJobsConverter extends ResourceConverter.IntegerConverter {
     public LocalTestJobsConverter() throws OptionsParsingException {
-      super(/* autoSupplier= */ () -> 0, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
+      super(/* auto= */ () -> 0, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
     }
   }
 

src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningOptions.java

@@ -13,7 +13,6 @@
 // limitations under the License.
 package com.google.devtools.build.lib.includescanning;
 
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
@@ -30,13 +29,9 @@ public class IncludeScanningOptions extends OptionsBase {
    * Converter for scanning parallelism threads: Takes {@value #FLAG_SYNTAX} 0 disables scanning
    * parallelism.
    */
-  public static class ParallelismConverter extends ResourceConverter {
+  public static class ParallelismConverter extends ResourceConverter.IntegerConverter {
     public ParallelismConverter() throws OptionsParsingException {
-      super(
-          /* autoSupplier= */ () ->
-              (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
-          /* minValue= */ 0,
-          /* maxValue= */ Integer.MAX_VALUE);
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
     }
   }
 

src/main/java/com/google/devtools/build/lib/pkgcache/PackageOptions.java

@@ -19,7 +19,6 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.PackageIdentifier;
 import com.google.devtools.build.lib.packages.RuleVisibility;
@@ -59,13 +58,9 @@ public String getTypeDescription() {
   }
 
   /** Converter for globbing threads. */
-  public static class ParallelismConverter extends ResourceConverter {
+  public static class ParallelismConverter extends ResourceConverter.IntegerConverter {
     public ParallelismConverter() throws OptionsParsingException {
-      super(
-          /* autoSupplier= */ () ->
-              (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
-          /* minValue= */ 1,
-          /* maxValue= */ Integer.MAX_VALUE);
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
     }
   }
 

src/main/java/com/google/devtools/build/lib/runtime/LoadingPhaseThreadsOption.java

@@ -14,7 +14,6 @@
 package com.google.devtools.build.lib.runtime;
 
 import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.build.lib.util.TestType;
 import com.google.devtools.common.options.Option;
@@ -45,18 +44,19 @@ public class LoadingPhaseThreadsOption extends OptionsBase {
   /**
    * A converter for loading phase thread count. Takes {@value FLAG_SYNTAX}. Caps at 20 for tests.
    */
-  public static final class LoadingPhaseThreadCountConverter extends ResourceConverter {
+  public static final class LoadingPhaseThreadCountConverter
+      extends ResourceConverter.IntegerConverter {
 
     public LoadingPhaseThreadCountConverter() {
       // TODO(jmmv): Using the number of cores has proven to yield reasonable analysis times on
       // Mac Pros and MacBook Pros but we should probably do better than this. (We haven't made
       // any guarantees that "auto" means number of cores precisely to leave us room to tune this
       // further in the future.)
-      super(() -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()));
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
     }
 
     @Override
-    public int checkAndLimit(int value) throws OptionsParsingException {
+    public Integer checkAndLimit(Integer value) throws OptionsParsingException {
       // Cap thread count while running tests. Test cases are typically small and large thread
       // pools vying for a relatively small number of CPU cores may induce non-optimal
       // performance.

src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java

@@ -18,7 +18,6 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.OptionsUtils;
 import com.google.devtools.build.lib.util.RamResourceConverter;
 import com.google.devtools.build.lib.util.ResourceConverter;
@@ -371,12 +370,9 @@ public ImmutableSet<Path> getInaccessiblePaths(FileSystem fs) {
   public int memoryLimitMb;
 
   /** Converter for the number of threads used for asynchronous tree deletion. */
-  public static final class AsyncTreeDeletesConverter extends ResourceConverter {
+  public static final class AsyncTreeDeletesConverter extends ResourceConverter.IntegerConverter {
     public AsyncTreeDeletesConverter() {
-      super(
-          () -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
-          0,
-          Integer.MAX_VALUE);
+      super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
     }
   }
 }

src/main/java/com/google/devtools/build/lib/util/BUILD

@@ -127,6 +127,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
         "//src/main/java/com/google/devtools/common/options",
+        "//third_party:error_prone_annotations",
         "//third_party:guava",
         "//third_party:jsr305",
     ],
@@ -139,7 +140,6 @@ java_library(
     ],
     deps = [
         ":resource_converter",
-        "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
         "//third_party:guava",
     ],
 )
@@ -151,7 +151,6 @@ java_library(
     ],
     deps = [
         ":resource_converter",
-        "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
         "//third_party:guava",
     ],
 )