diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 7d8a72e..cf858ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -388,7 +388,7 @@ public QueueInfo getQueueInfo( } @Override - public synchronized List + public List getQueueUserAclInfo(UserGroupInformation user) { QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(QueueUserACLInfo.class); @@ -889,7 +889,8 @@ private void killToPreemptContainers(Resource clusterResource, } } - private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { + @VisibleForTesting + protected void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { // Set preemption-allowed: // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 19003b5..18527cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.User; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSchedulerKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; @@ -96,6 +97,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doAnswer; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSchedulerKey; public class TestLeafQueue { @@ -1687,7 +1691,137 @@ public void testReservationExchange() throws Exception { assertEquals(0*GB, app_1.getCurrentReservation().getMemorySize()); assertEquals(0*GB, node_0.getUsedResource().getMemorySize()); } - + + /** + * Test deadlock when ParentQueue#getQueueUserAclInfo is called by Thread_A + * at the moment that Thread_B calls LeafQueue#assignContainers to + * release a reserved container. + */ + @Test(timeout = 60000) + public void testRMDeadlock() throws Exception { + + // Manipulate queue 'a' + LeafQueue a = (LeafQueue) queues.get(A); + //unset maxCapacity + a.setMaxCapacity(1.0f); + a.setUserLimitFactor(10); + + // Users + final String user_0 = "user_0"; + final String user_1 = "user_1"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, + mock(ActiveUsersManager.class), spyRMContext); + a.submitApplicationAttempt(app_0, user_0); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a, + mock(ActiveUsersManager.class), spyRMContext); + a.submitApplicationAttempt(app_1, user_1); + + // Setup some nodes + String host_0 = "127.0.0.1"; + FiCaSchedulerNode node_0 = + TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 4 * GB); + + String host_1 = "127.0.0.2"; + FiCaSchedulerNode node_1 = + TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 4 * GB); + + when(csContext.getNode(node_0.getNodeID())).thenReturn(node_0); + when(csContext.getNode(node_1.getNodeID())).thenReturn(node_1); + + final int numNodes = 3; + Resource clusterResource = + Resources.createResource(numNodes * (4 * GB), numNodes * 16); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + when(csContext.getMaximumResourceCapability()) + .thenReturn(Resources.createResource(4 * GB, 16)); + when(a.getMaximumAllocation()) + .thenReturn(Resources.createResource(4 * GB, 16)); + when(a.getMinimumAllocationFactor()).thenReturn(0.25f); // 1G / 4G + + // Setup resource-requests + Priority priority = TestUtils.createMockPriority(1); + app_0.updateResourceRequests(Collections.singletonList(TestUtils + .createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, + priority, recordFactory))); + + app_1.updateResourceRequests(Collections.singletonList(TestUtils + .createResourceRequest(ResourceRequest.ANY, 4 * GB, 1, true, + priority, recordFactory))); + + // Start testing... + + // allocate 1 container for app_0 on node_0 + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + + // Now, reservation should kick in for app_1 on node_0 + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + + // re-reserve twice for app_1 + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + + assertEquals(5 * GB, a.getUsedResources().getMemorySize()); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize()); + assertEquals(4 * GB, app_1.getCurrentReservation().getMemorySize()); + assertEquals(2, app_1.getReReservations(toSchedulerKey(priority))); + + // assign new container for app_1 on node_1 + a.assignContainers(clusterResource, node_1, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + assertEquals(9 * GB, a.getUsedResources().getMemorySize()); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(4 * GB, app_1.getCurrentConsumption().getMemorySize()); + assertEquals(4 * GB, app_1.getCurrentReservation().getMemorySize()); + assertEquals(1 * GB, node_0.getUsedResource().getMemorySize()); + assertEquals(4 * GB, node_1.getUsedResource().getMemorySize()); + + // Test deadlock + final CSQueue parentQueue = a.getParent(); + final UserGroupInformation user = UserGroupInformation.getCurrentUser(); + doAnswer(new Answer() { + @Override public Void answer(InvocationOnMock invocationOnMock) + throws Throwable { + Object[] args = invocationOnMock.getArguments(); + Thread t = new Thread(new Runnable() { + @Override public void run() { + parentQueue.getQueueUserAclInfo(user); + } + }); + t.start(); + Thread.sleep(1000); + return null; + } + }).when(a) + .setPreemptionAllowed(any(ResourceLimits.class), any(String.class)); + // release reserved container for app_1 on node_0 + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + assertEquals(5 * GB, a.getUsedResources().getMemorySize()); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(4 * GB, app_1.getCurrentConsumption().getMemorySize()); + assertEquals(0 * GB, app_1.getCurrentReservation().getMemorySize()); + } + private void verifyContainerAllocated(CSAssignment assignment, NodeType nodeType) { Assert.assertTrue(Resources.greaterThan(resourceCalculator, null, assignment.getResource(), Resources.none()));