Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Added config toggle for verbose GC logging #3663

Merged
merged 17 commits into from
Jan 21, 2021
Merged

Added config toggle for verbose GC logging #3663

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
9624546
Added config toggle for verbose GC logging
nicknezis Jan 12, 2021
3de59e3
Fixed typo
nicknezis Jan 12, 2021
1c29e0a
Update heron/executor/src/python/heron_executor.py
nicknezis Jan 12, 2021
e4fe1dd
Update heron/executor/src/python/heron_executor.py
nicknezis Jan 12, 2021
b6da4b6
Update heron/executor/src/python/heron_executor.py
nicknezis Jan 12, 2021
a2954e1
Update heron/executor/src/python/heron_executor.py
nicknezis Jan 12, 2021
9a1fff6
Updated the logic so that it can be configured at submission time
nicknezis Jan 13, 2021
a3eabf4
Style fix
nicknezis Jan 13, 2021
f7bca95
Style fix
nicknezis Jan 13, 2021
a57c798
Updated heron_executor_unittest
nicknezis Jan 14, 2021
d47c857
Removing unnecessary comments
nicknezis Jan 14, 2021
eaa23ec
Added verbose-gc flag to the Heron CLI
nicknezis Jan 14, 2021
a95896c
Adding missing argument
nicknezis Jan 14, 2021
afc3721
Forgot to make the argument a boolean
nicknezis Jan 14, 2021
f8797a0
Correcting type
nicknezis Jan 14, 2021
de0e609
More updates
nicknezis Jan 20, 2021
4f73981
Style cleanup
nicknezis Jan 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 13 additions & 30 deletions heron/executor/src/python/heron_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@
@click.option("--topology-defn-file", required=True)
@click.option("--topology-id", required=True)
@click.option("--topology-name", required=True)
@click.option("--verbose-gc", is_flag=True)
@click.option("--jvm-remote-debugger-ports",
help="comma separated list of ports to be used"
" by a remote debugger for JVM instances")
Expand Down Expand Up @@ -342,6 +343,7 @@ def init_from_parsed_args(self, parsed_args):
self.health_manager_mode = parsed_args.health_manager_mode
self.health_manager_classpath = '%s:%s'\
% (self.scheduler_classpath, parsed_args.health_manager_classpath)
self.verbose_gc = parsed_args.verbose_gc
self.jvm_remote_debugger_ports = \
parsed_args.jvm_remote_debugger_ports.split(",") \
if parsed_args.jvm_remote_debugger_ports else None
Expand Down Expand Up @@ -574,36 +576,17 @@ def _get_java_major_version(self):
return int(self._get_jvm_version().split(".")[0])

def _get_java_gc_instance_cmd(self, cmd, gc_name):
gc_cmd = ['-verbosegc']
if self._get_java_major_version() >= 9:
gc_cmd += [
'-XX:+UseG1GC',
'-XX:+ParallelRefProcEnabled',
'-XX:+UseStringDeduplication',
'-XX:MaxGCPauseMillis=100',
'-XX:InitiatingHeapOccupancyPercent=30',
'-XX:+HeapDumpOnOutOfMemoryError',
'-XX:ParallelGCThreads=4',
'-Xlog:gc*,safepoint=info:file=' + self.log_dir + '/gc.' + gc_name +
'.log:tags,time,uptime,level:filecount=5,filesize=100M']
else:
gc_cmd += [
'-XX:+UseConcMarkSweepGC',
'-XX:+CMSScavengeBeforeRemark',
'-XX:TargetSurvivorRatio=90',
'-XX:+PrintGCDetails',
'-XX:+PrintGCTimeStamps',
'-XX:+PrintGCDateStamps',
'-XX:+PrintGCCause',
'-XX:+UseGCLogFileRotation',
'-XX:NumberOfGCLogFiles=5',
'-XX:GCLogFileSize=100M',
'-XX:+PrintPromotionFailure',
'-XX:+PrintTenuringDistribution',
'-XX:+PrintHeapAtGC',
'-XX:+HeapDumpOnOutOfMemoryError',
'-XX:ParallelGCThreads=4',
'-Xloggc:' + self.log_dir + '/gc.' + gc_name + '.log']
gc_cmd = [
'-XX:+UseG1GC',
'-XX:+ParallelRefProcEnabled',
'-XX:+UseStringDeduplication',
'-XX:MaxGCPauseMillis=100',
'-XX:InitiatingHeapOccupancyPercent=30',
'-XX:+HeapDumpOnOutOfMemoryError',
'-XX:ParallelGCThreads=4']
if self.verbose_gc:
gc_cmd += ['-Xlog:gc*,safepoint=info:file=' + self.log_dir + '/gc.' + gc_name +
'.log:tags,time,uptime,level:filecount=5,filesize=100M']
try:
cp_index = cmd.index('-cp')
return list(itertools.chain(*[cmd[0:cp_index], gc_cmd, cmd[cp_index:]]))
Expand Down
158 changes: 23 additions & 135 deletions heron/executor/tests/python/heron_executor_unittest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def _run_process(self, name, cmd, env=None):
return popen

def _get_jvm_version(self):
return "1.8.y.x"
return "11.0.6"


class HeronExecutorTest(unittest.TestCase):
Expand All @@ -105,50 +105,41 @@ def build_packing_plan(self, instance_distribution):
instance_plan.component_index = int(component_index)
return packing_plan

# pylint: disable=no-self-argument
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.metricsmgr-%d.log " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"-Xloggc:log-files" might be necessary.

"-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
"--id=metricsmgr-%d --port=metricsmgr_port " \
"--topology=topname --cluster=cluster --role=role --environment=environ " \
"--topology-id=topid " \
"--system-config-file=%s --override-config-file=%s " \
"--sink-config-file=metrics_sinks_config_file" %\
(container_id, container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
"--sink-config-file=metrics_sinks_config_file" % \
(container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)

def get_expected_metricscachemgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.metricscache.log " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"-Xloggc:log-files" might be necessary.

"-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
"--metricscache_id metricscache-0 --server_port metricscachemgr_serverport " \
"--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
"--system_config_file %s --override_config_file %s " \
"--sink_config_file metrics_sinks_config_file " \
"--cluster cluster --role role --environment environ" %\
(INTERNAL_CONF_PATH, OVERRIDE_PATH)
"--cluster cluster --role role --environment environ" \
% (INTERNAL_CONF_PATH, OVERRIDE_PATH)

def get_expected_healthmgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.healthmgr.log " \
"-Djava.net.preferIPv4Stack=true " \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"-Xloggc:log-files" might be necessary.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as comment above. I believe -Xloggc:log-files is a GC specific setting. This does not impact the default logging.

"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-cp scheduler_classpath:healthmgr_classpath " \
"org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
"--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
Expand All @@ -157,19 +148,16 @@ def get_expected_instance_command(component_name, instance_id, container_id):
instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
"-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
"-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
"-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
"-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
"-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.%s.log " \
"-Djava.net.preferIPv4Stack=true " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"Xloggc:log-files/gc.%s.log" and the instance_name are still necessary I feel since all the logs are in the same folder.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-Xloggc set of flags are all related to the GC logging. In my testing, I still saw JVM component log files in the log-files folder. This change should only remove the GC specific logging (unless someone chooses to enable at submit time).

I believe if the verbose-gc flag is not set, the changes I made to the code will not set those JVM flags. This unit test is testing with the defaults, so what you see here matches what was generated by the heron_executor.py.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. The flag is for gc logs only, but it separates different gc logs into different files. For example, the gc log for this process would be "log-files/gc.healthmgr.log" and each process has its own gc log file.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the heron_executor functionality that was setting that path and log file names is still there. It's just not there for the default submission. You have to use the flag to enable those parameters.

These lines of code that you are commenting on are test results for the default case that does not have verbose_gc enabled. If there was a test for the verbose_gc use case, then we would have another version of this string that included the various GC log parameters. I'm not sure how much more work it will be to add another test case that captures that scenario, but I will look into making it.

"-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
"org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
"-instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
"-stmgr_port tmanager_controller_port -metricsmgr_port metricsmgr_port " \
"-system_config_file %s -override_config_file %s" \
% (instance_name, instance_name, component_name, instance_id,
% (instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)

MockPOpen.set_next_pid(37)
Expand Down Expand Up @@ -391,103 +379,3 @@ def assert_process(self, expected_process, found_processes):
self.assertEqual(expected_process.command, found_processes[pid].command_str)
self.assertEqual(1, found_processes[pid].attempts)


class MockExecutorJDK11(HeronExecutor):
"""
mock executor that overrides methods that don't apply to unit tests, like running processes
"""
def __init__(self, args):
self.processes = []
super(MockExecutorJDK11, self).__init__(args, None)

# pylint: disable=no-self-use
def _load_logging_dir(self, heron_internals_config_file):
return "log-files"

def _run_process(self, name, cmd, env=None):
popen = MockPOpen()
self.processes.append(ProcessInfo(popen, name, cmd))
return popen

def _get_jvm_version(self):
return "11.0.6"


class HeronExecutorJDK11Test(unittest.TestCase):
"""Unittest for Heron Executor"""

def __init__(self, args):
super(HeronExecutorJDK11Test, self).__init__(args, None)

# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xlog:gc*,safepoint=info:file=log-files/gc.metricsmgr-%d.log:tags,time,uptime," \
"level:filecount=5,filesize=100M " \
"-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
"--id=metricsmgr-%d --port=metricsmgr_port " \
"--topology=topname --cluster=cluster --role=role --environment=environ " \
"--topology-id=topid " \
"--system-config-file=%s --override-config-file=%s " \
"--sink-config-file=metrics_sinks_config_file" % \
(container_id, container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)

def get_expected_metricscachemgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
" -Xlog:gc*,safepoint=info:file=log-files/gc.metricscache.log:tags,time,uptime," \
"level:filecount=5,filesize=100M " \
"-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
"--metricscache_id metricscache-0 --server_port metricscachemgr_serverport " \
"--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
"--system_config_file %s --override_config_file %s " \
"--sink_config_file metrics_sinks_config_file " \
"--cluster cluster --role role --environment environ" % \
(INTERNAL_CONF_PATH, OVERRIDE_PATH)

def get_expected_healthmgr_command():
return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
" -Xlog:gc*,safepoint=info:file=log-files/gc.healthmgr.log:tags,time,uptime," \
"level:filecount=5,filesize=100M " \
"-cp scheduler_classpath:healthmgr_classpath " \
"org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
"--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"

def get_expected_instance_command(component_name, instance_id, container_id):
instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
"-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
"-Djava.net.preferIPv4Stack=true -verbosegc " \
"-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
"-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
"-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
"-Xloggc:log-files/gc.%s.log " \
"-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
"org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
"-instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
"-stmgr_port tmanager_controller_port -metricsmgr_port metricsmgr_port " \
"-system_config_file %s -override_config_file %s" \
% (instance_name, instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)

def setUp(self):
MockPOpen.set_next_pid(37)
self.maxDiff = None
self.executor_0 = MockExecutorJDK11(self.get_args(0))
self.executor_1 = MockExecutorJDK11(self.get_args(1))
self.executor_7 = MockExecutorJDK11(self.get_args(7))
self.packing_plan_expected = self.build_packing_plan({
1:[('word', '3', '0'), ('exclaim1', '2', '0'), ('exclaim1', '1', '0')],
7:[('word', '11', '0'), ('exclaim1', '210', '0')],
})
1 change: 1 addition & 0 deletions heron/scheduler-core/src/java/org/apache/heron/scheduler/ExecutorFlag.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public enum ExecutorFlag {
StatefulConfigFile("stateful-config-file"),
HealthManagerMode("health-manager-mode"),
HealthManagerClasspath("health-manager-classpath"),
EnableVerboseGCLog("verbose-gc"),
JvmRemoteDebuggerPorts("jvm-remote-debugger-ports");

private final String name;
Expand Down
17 changes: 15 additions & 2 deletions heron/scheduler-core/src/java/org/apache/heron/scheduler/SubmitterMain.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public class SubmitterMain {
* @param dryRun run as dry run
* @param dryRunFormat the dry run format
* @param verbose enable verbose logging
* @param verboseGC enable verbose JVM GC logging
* @return config the command line config
*/
protected static Config commandLineConfigs(String cluster,
Expand All @@ -81,7 +82,8 @@ protected static Config commandLineConfigs(String cluster,
String submitUser,
Boolean dryRun,
DryRunFormatType dryRunFormat,
Boolean verbose) {
Boolean verbose,
Boolean verboseGC) {
return Config.newBuilder()
.put(Key.CLUSTER, cluster)
.put(Key.ROLE, role)
Expand All @@ -90,6 +92,7 @@ protected static Config commandLineConfigs(String cluster,
.put(Key.DRY_RUN, dryRun)
.put(Key.DRY_RUN_FORMAT_TYPE, dryRunFormat)
.put(Key.VERBOSE, verbose)
.put(Key.VERBOSE_GC, verboseGC)
.build();
}

Expand Down Expand Up @@ -207,6 +210,11 @@ private static Options constructOptions() {
.longOpt("verbose")
.build();

Option verboseGC = Option.builder("vgc")
.desc("Enable verbose JVM GC logs")
.longOpt("verboseGC")
.build();

options.addOption(cluster);
options.addOption(role);
options.addOption(environment);
Expand All @@ -221,6 +229,7 @@ private static Options constructOptions() {
options.addOption(dryRun);
options.addOption(dryRunFormat);
options.addOption(verbose);
options.addOption(verboseGC);

return options;
}
Expand All @@ -241,6 +250,10 @@ private static boolean isVerbose(CommandLine cmd) {
return cmd.hasOption("v");
}

private static boolean isVerboseGC(CommandLine cmd) {
return cmd.hasOption("vgc");
}

@SuppressWarnings("JavadocMethod")
@VisibleForTesting
public static Config loadConfig(CommandLine cmd, TopologyAPI.Topology topology) {
Expand Down Expand Up @@ -277,7 +290,7 @@ public static Config loadConfig(CommandLine cmd, TopologyAPI.Topology topology)
return Config.toLocalMode(Config.newBuilder()
.putAll(ConfigLoader.loadConfig(heronHome, configPath, releaseFile, overrideConfigFile))
.putAll(commandLineConfigs(cluster, role, environ, submitUser, dryRun,
dryRunFormat, isVerbose(cmd)))
dryRunFormat, isVerbose(cmd), isVerboseGC(cmd)))
.putAll(SubmitterUtils.topologyConfigs(topologyPackage, topologyBinaryFile,
topologyDefnFile, topology))
.build());
Expand Down
3 changes: 3 additions & 0 deletions heron/scheduler-core/src/java/org/apache/heron/scheduler/utils/SchedulerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,9 @@ public static void addExecutorTopologyArgs(List<String> args, Config config, Con
args.add(createCommandArg(ExecutorFlag.TopologyBinaryFile,
Context.topologyBinaryFile(config)));
args.add(createCommandArg(ExecutorFlag.HeronJavaHome, Context.clusterJavaHome(config)));
if (Context.verboseGC(config)) {
args.add(ExecutorFlag.EnableVerboseGCLog.getFlag());
}
args.add(createCommandArg(ExecutorFlag.HeronShellBinary, Context.shellBinary(config)));
args.add(createCommandArg(ExecutorFlag.Cluster, Context.cluster(config)));
args.add(createCommandArg(ExecutorFlag.Role, Context.role(config)));
Expand Down
4 changes: 4 additions & 0 deletions heron/spi/src/java/org/apache/heron/spi/common/Context.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ public static Boolean verbose(Config cfg) {
return cfg.getBooleanValue(Key.VERBOSE);
}

public static Boolean verboseGC(Config cfg) {
return cfg.getBooleanValue(Key.VERBOSE_GC);
}

public static String buildVersion(Config cfg) {
return cfg.getStringValue(Key.BUILD_VERSION);
}
Expand Down
3 changes: 3 additions & 0 deletions heron/spi/src/java/org/apache/heron/spi/common/Key.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ public enum Key {
DRY_RUN ("heron.config.dry_run", Boolean.FALSE),
DRY_RUN_FORMAT_TYPE ("heron.config.dry_run_format_type", Type.DRY_RUN_FORMAT_TYPE),
VERBOSE ("heron.config.verbose", Boolean.FALSE),
// Used to enable verbose JVM GC logging
VERBOSE_GC ("heron.config.verbose.gc", Boolean.FALSE),

CONFIG_PROPERTY ("heron.config.property", Type.STRING),

//keys for release/build information
Expand Down
11 changes: 11 additions & 0 deletions heron/tools/cli/src/python/args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,17 @@ def add_verbose(parser):
help='Verbose mode. Increases logging level to show debug messages')
return parser

def add_verbose_gc(parser):
'''
:param parser:
:return:
'''
parser.add_argument(
'--verbose-gc',
default=False,
action='store_true',
help='Produce JVM GC logging')
return parser

def add_topology(parser):
'''
Expand Down
Loading