From c2509752c4f14a903aaabdf0c5265361a6f10b09 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Fri, 8 Mar 2019 17:37:15 +0100 Subject: [PATCH] YARN-9337. GPU auto-discovery script runs even when the resource is given by hand --- .../resourceplugin/gpu/GpuDiscoverer.java | 1 + .../resourceplugin/gpu/TestGpuDiscoverer.java | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java index 6cf6a8dbbf276dd9d8877923226923d69973f3c6..e82782d2e25038cd1c5f48f5cecab7984846c2ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java index ff64e042e19f526ae917c971111fa4a678b1da16..0f67214348235d298e2ad011a92896f6869191a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java @@ -40,6 +40,7 @@ import java.util.function.Consumer; import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_GPU_ALLOWED_DEVICES; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer.DEFAULT_BINARY_NAME; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; @@ -49,6 +50,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.*; public class TestGpuDiscoverer { private static final Logger LOG = LoggerFactory.getLogger( @@ -96,7 +98,7 @@ public void before() throws IOException { private Configuration createConfigWithAllowedDevices(String s) { Configuration conf = new Configuration(false); - conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, s); + conf.set(NM_GPU_ALLOWED_DEVICES, s); setupFakeBinary(conf); return conf; } @@ -495,4 +497,17 @@ public void testGpuBinaryIsANotExistingFile() { "executable in the default directories:")); } } + + @Test + public void testScriptNotCalled() throws YarnException { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:1,2:3"); + + GpuDiscoverer gpuSpy = spy(GpuDiscoverer.class); + + gpuSpy.initialize(conf); + gpuSpy.getGpusUsableByYarn(); + + verify(gpuSpy, never()).getGpuDeviceInformation(); + } } -- 2.17.2 (Apple Git-113)