From 70f0900696e4c73082de0dc4d59479d3412c2187 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Thu, 18 Jul 2019 11:15:59 +0200 Subject: [PATCH] YARN-9375. Use Configured in GpuDiscoverer and FpgaDiscoverer --- .../resourceplugin/fpga/FpgaDiscoverer.java | 17 ++++++----------- .../resourceplugin/gpu/GpuDiscoverer.java | 12 ++++++------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java index 180a011b61f59..afe190a603812 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -46,11 +47,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -public class FpgaDiscoverer { +public class FpgaDiscoverer extends Configured { private static final Logger LOG = LoggerFactory.getLogger( FpgaDiscoverer.class); - private Configuration conf = null; private AbstractFpgaVendorPlugin plugin = null; private List currentFpgaInfo = null; @@ -64,11 +64,6 @@ void setScriptRunner(Function> scriptRunner) { this.scriptRunner = scriptRunner; } - @VisibleForTesting - public void setConf(Configuration configuration) { - this.conf = configuration; - } - public List getCurrentFpgaInfo() { return currentFpgaInfo; } @@ -82,7 +77,7 @@ public boolean diagnose() { } public void initialize(Configuration config) throws YarnException { - this.conf = config; + setConf(config); this.plugin.initPlugin(config); // Try to diagnose FPGA LOG.info("Trying to diagnose FPGA information ..."); @@ -100,11 +95,11 @@ public void initialize(Configuration config) throws YarnException { public List discover() throws ResourceHandlerException { List list; - String allowed = this.conf.get(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES); + String allowed = getConf().get(YarnConfiguration.NM_FPGA_ALLOWED_DEVICES); - String availableDevices = conf.get( + String availableDevices = getConf().get( YarnConfiguration.NM_FPGA_AVAILABLE_DEVICES); - String discoveryScript = conf.get( + String discoveryScript = getConf().get( YarnConfiguration.NM_FPGA_DEVICE_DISCOVERY_SCRIPT); FPGADiscoveryStrategy discoveryStrategy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java index 986f84a48f3f5..939093f40305d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -45,7 +46,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable -public class GpuDiscoverer { +public class GpuDiscoverer extends Configured { public static final Logger LOG = LoggerFactory.getLogger( GpuDiscoverer.class); @VisibleForTesting @@ -61,7 +62,6 @@ public class GpuDiscoverer { private static final int MAX_EXEC_TIMEOUT_MS = 10 * 1000; private static final int MAX_REPEATED_ERROR_ALLOWED = 10; - private Configuration conf = null; private String pathOfGpuBinary = null; private Map environment = new HashMap<>(); @@ -71,7 +71,7 @@ public class GpuDiscoverer { private List gpuDevicesFromUser; private void validateConfOrThrowException() throws YarnException { - if (conf == null) { + if (getConf() == null) { throw new YarnException("Please initialize (call initialize) before use " + GpuDiscoverer.class.getSimpleName()); } @@ -144,7 +144,7 @@ synchronized GpuDeviceInformation getGpuDeviceInformation() } private boolean IsAutoDiscoveryEnabled() { - String allowedDevicesStr = conf.get( + String allowedDevicesStr = getConf().get( YarnConfiguration.NM_GPU_ALLOWED_DEVICES, YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES); return allowedDevicesStr.equals( @@ -205,7 +205,7 @@ private List parseGpuDevicesFromAutoDiscoveredGpuInfo() */ private List parseGpuDevicesFromUserDefinedValues() throws YarnException { - String devices = conf.get( + String devices = getConf().get( YarnConfiguration.NM_GPU_ALLOWED_DEVICES, YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES); @@ -249,7 +249,7 @@ private GpuDevice parseGpuDevice(String device, String[] splitByColon, public synchronized void initialize(Configuration config) throws YarnException { - this.conf = config; + setConf(config); if (IsAutoDiscoveryEnabled()) { numOfErrorExecutionSinceLastSucceed = 0; lookUpAutoDiscoveryBinary(config);