Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Add driver memory config for yarn scheduler #1454

Merged
merged 1 commit into from
Sep 29, 2016
Merged

Add driver memory config for yarn scheduler #1454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions heron/config/src/yaml/conf/yarn/scheduler.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,6 @@ heron.directory.sandbox.java.home: ${JAVA_HOME}

# yarn queue for submitting and launching the topology
heron.scheduler.yarn.queue: default

# the amount of memory topology's driver (yarn application master) needs
heron.scheduler.yarn.driver.memory.mb: 1024
9 changes: 8 additions & 1 deletion heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,19 @@
import com.twitter.heron.spi.common.Context;

public final class YarnContext extends Context {
public static final String HERON_SCHEDULER_YARN_QUEUE = "heron.scheduler.yarn.queue";
private static final String PREFIX = "heron.scheduler.yarn.";

public static final String HERON_SCHEDULER_YARN_QUEUE = PREFIX + "queue";
public static final String YARN_SCHEDULER_DRIVER_MEMORY_MB = PREFIX + "driver.memory.mb";

private YarnContext() {
}

public static String heronYarnQueue(Config cfg) {
return cfg.getStringValue(HERON_SCHEDULER_YARN_QUEUE, "default");
}

public static int heronDriverMemoryMb(Config cfg) {
return cfg.getIntegerValue(YARN_SCHEDULER_DRIVER_MEMORY_MB, 1024);
}
}
3 changes: 3 additions & 0 deletions heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public class YarnLauncher implements ILauncher {
private String role;
private String env;
private String queue;
private int driverMemory;
private ArrayList<String> libJars = new ArrayList<>();

@Override
Expand All @@ -72,6 +73,7 @@ public void initialize(Config config, Config runtime) {
role = Context.role(config);
env = Context.environ(config);
queue = YarnContext.heronYarnQueue(config);
driverMemory = YarnContext.heronDriverMemoryMb(config);

try {
Class<?> packingClass = Class.forName(Context.packingClass(config));
Expand Down Expand Up @@ -153,6 +155,7 @@ private Configuration getHMDriverConf() {
.set(HeronDriverConfiguration.HTTP_PORT, 0)
.set(HeronDriverConfiguration.VERBOSE, false)
.set(YarnDriverConfiguration.QUEUE, queue)
.set(DriverConfiguration.DRIVER_MEMORY, driverMemory)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be for another PR, but it would be great to have a unit test that loads things up and verifies the default configs, then another one that loads things up with overrides and verifies that too. Such a test would verify that this .set call wasn't overlooked.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. I have some ideas to test the yarnconfigs. Will create an issue to track it separately. Thanks.

.build();
}

Expand Down