diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 258c7dc..5fdcf15 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -169,6 +169,23 @@ public RegisterNodeManagerResponse registerNodeManager( return response; } + // Check if this node has minimum allocations + Configuration conf = getConfig(); + int minAllocMb = conf.getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); + int minAllocVcores = conf.getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + if (capability.getMemory() < minAllocMb + || capability.getVirtualCores() < minAllocVcores) { + LOG.info("NodeManager from " + host + + " does't satisfy minimum allocations, Sending SHUTDOWN" + + " signal to the NodeManager."); + response.setNodeAction(NodeAction.SHUTDOWN); + return response; + } + if (isSecurityEnabled()) { MasterKey nextMasterKeyForNode = this.containerTokenSecretManager.getCurrentKey(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index a2f59ef..2de614d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -45,6 +45,7 @@ private int responseId; private NodeId nodeId; private final int memory; + private final int vCores = 1; private ResourceTrackerService resourceTracker; private final int httpPort = 2; private MasterKey currentMasterKey; @@ -53,9 +54,7 @@ public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTrack this.memory = memory; this.resourceTracker = resourceTracker; String[] splits = nodeIdStr.split(":"); - nodeId = Records.newRecord(NodeId.class); - nodeId.setHost(splits[0]); - nodeId.setPort(Integer.parseInt(splits[1])); + nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1])); } public NodeId getNodeId() { @@ -83,8 +82,7 @@ public RegisterNodeManagerResponse registerNode() throws Exception { RegisterNodeManagerRequest.class); req.setNodeId(nodeId); req.setHttpPort(httpPort); - Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + Resource resource = BuilderUtils.newResource(memory, vCores); req.setResource(resource); RegisterNodeManagerResponse registrationResponse = resourceTracker.registerNodeManager(req); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index af9d5d2..25961fd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Test; @@ -268,6 +270,49 @@ public void testNodeRegistrationFailure() throws Exception { } @Test + public void testNodeRegistrationWithMinimumAllocations() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048"); + conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4"); + rm = new MockRM(conf); + rm.start(); + + ResourceTrackerService resourceTrackerService + = rm.getResourceTrackerService(); + RegisterNodeManagerRequest req = Records.newRecord( + RegisterNodeManagerRequest.class); + NodeId nodeId = BuilderUtils.newNodeId("host", 1234); + req.setNodeId(nodeId); + + Resource capability = BuilderUtils.newResource(1024, 1); + req.setResource(capability); + RegisterNodeManagerResponse response1 = + resourceTrackerService.registerNodeManager(req); + Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction()); + + capability.setMemory(2048); + capability.setVirtualCores(1); + req.setResource(capability); + RegisterNodeManagerResponse response2 = + resourceTrackerService.registerNodeManager(req); + Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction()); + + capability.setMemory(1024); + capability.setVirtualCores(4); + req.setResource(capability); + RegisterNodeManagerResponse response3 = + resourceTrackerService.registerNodeManager(req); + Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); + + capability.setMemory(2048); + capability.setVirtualCores(4); + req.setResource(capability); + RegisterNodeManagerResponse response4 = + resourceTrackerService.registerNodeManager(req); + Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction()); + } + + @Test public void testReboot() throws Exception { Configuration conf = new Configuration(); rm = new MockRM(conf); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index bf985d3..9f71c7e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Before; import org.junit.Test; @@ -127,7 +128,7 @@ public void testNMExpiry() throws Exception { String hostname1 = "localhost1"; String hostname2 = "localhost2"; String hostname3 = "localhost3"; - Resource capability = recordFactory.newRecordInstance(Resource.class); + Resource capability = BuilderUtils.newResource(1024, 1); RegisterNodeManagerRequest request1 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index 1fd1b2c..d4ac7b5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; @@ -89,7 +90,7 @@ public void tearDown() { @Test public void testRPCResponseId() throws IOException { String node = "localhost"; - Resource capability = recordFactory.newRecordInstance(Resource.class); + Resource capability = BuilderUtils.newResource(1024, 1); RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class); nodeId = Records.newRecord(NodeId.class); nodeId.setHost(node);