Improve heap usage configurations

.github/env/Linux/gradle.properties

@@ -4,11 +4,13 @@
 # GitHub hosted linux runner environment has 2 CPUs and 7G RAM.
 #
 # testParallel seems to run poorly on GH Actions, and is unable to take advantage it seems of 2
-# workers. Let's bump up max heap, and target it as the lowest-common denominator.
+# workers.
+#
+# Our engine-table tests currently request 6g of heap (engine/table/build.gradle); this means we can't have more than
+# 1 worker at a time.
 
 org.gradle.parallel=false
 org.gradle.workers.max=1
-org.gradle.jvmargs=-Xmx5g
 
 # Our CI JDKs should be pre-provisioned and invoked correctly,
 # we shouldn't rely on gradle for any of this logic.

Integrations/build.gradle

@@ -78,6 +78,7 @@ def runInDocker = { String name, String sourcePath, List<String> command, Closur
  volume '/data'
  volume '/cache'
  environmentVariable 'DEEPHAVEN_CLASSPATH', '/classpath/*:/classpath:/opt/deephaven/server/lib/*:/opt/deephaven/server/lib/'
+ environmentVariable 'DEEPHAVEN_MAXMEM', '2G'
  environmentVariable 'DEEPHAVEN_PROPFILE', 'dh-defaults.prop'
  environmentVariable 'DEEPHAVEN_VERSION', project.version
 

gradle.properties

@@ -3,8 +3,18 @@
 ## Note, you can use -Dorg.gradle.debug=true as well.
 ## Be sure to set remote debugger configuration to use the deephaven buildSrc project for classpath.
 
-# use this to control the jvm arguments of gradle daemons.
-org.gradle.jvmargs=-Xms1g -Xmx3g
+# The `org.gradle.jvmargs` property controls the memory settings for the gradle _daemon_.
+# The majority of "real" work is done by gradle _workers_.
+# The gradle daemon needs enough memory to handle the configuration and coordination for the build process.
+# By default, the gradle daemon sets 512m for max memory, but our project (currently) needs a little bit more.
+# The amount of memory a daemon uses is a function of how complex the project is, as well as the gradle implementation
+# details which may vary from release to release.
+# The gradle build scans provide a nice "Peak heap memory usage" value under the "Performance" tab which may help guide
+# setting this value.
+#
+# The gradle workers often have their own methods to control additional heap.
+# For example, compilation and java tests are able to set their own memory usage needs through their own configurations.
+org.gradle.jvmargs=-Xmx1g
 
 # Setting debugCI to true will cause failed builds to keepalive for three hours so we can do post-mortem inspection of jenkins workspace
 debugCI=false

py/server/test_helper/__init__.py

@@ -15,7 +15,7 @@
 py_dh_session = None
 
 
-def start_jvm(jvm_props: Dict[str, str] = None):
+def start_jvm_for_tests(jvm_props: Dict[str, str] = None):
  jvm.preload_jvm_dll()
  import jpy
 
@@ -52,10 +52,13 @@ def start_jvm(jvm_props: Dict[str, str] = None):
  }
  jvm_classpath = os.environ.get('DEEPHAVEN_CLASSPATH', '')
 
+ # Intentionally small by default - callers should set as appropriate
+ jvm_maxmem = os.environ.get('DEEPHAVEN_MAXMEM', '256m')
+
  # Start up the JVM
  jpy.VerboseExceptions.enabled = True
  jvm.init_jvm(
- jvm_maxmem='1G',
+ jvm_maxmem=jvm_maxmem,
  jvm_classpath=_expand_wildcards_in_list(jvm_classpath.split(os.path.pathsep)),
  jvm_properties=jvm_properties,
  jvm_options=jvm_options

py/server/tests/__init__.py

@@ -2,5 +2,6 @@
 # Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
 #
 
-from test_helper import start_jvm
-start_jvm()
+from test_helper import start_jvm_for_tests
+
+start_jvm_for_tests()