diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index 6dcecde09aa..c05f7acfd28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -1037,6 +1037,11 @@ private void updateAMRMToken(Token token) throws IOException { return remoteRequests.get(Long.valueOf(allocationRequestId)); } + @VisibleForTesting + Map, List> getOutstandingSchedRequests() { + return outstandingSchedRequests; + } + RemoteRequestsTable putTable(long allocationRequestId, RemoteRequestsTable table) { return remoteRequests.put(Long.valueOf(allocationRequestId), table); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java index 0e882996d42..6552b459c8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java @@ -133,18 +133,26 @@ public float getProgress() { schedulingRequest(1, 1, 3, 1, 512, "foo"), schedulingRequest(1, 1, 4, 1, 512, "foo"), // 1 req with numAlloc = 4 - schedulingRequest(4, 1, 5, 1, 512, "bar"))); + schedulingRequest(4, 1, 5, 1, 512, "bar"), + // 1 empty tag + schedulingRequest(1, 1, 6, 1, 512, null))); // kick the scheduler waitForContainerAllocation(allocatedContainers, - rejectedSchedulingRequests, 6, 2); + rejectedSchedulingRequests, 7, 2); - Assert.assertEquals(6, allocatedContainers.size()); + Assert.assertEquals(7, allocatedContainers.size()); Map> containersPerNode = allocatedContainers.stream().collect( Collectors.groupingBy(Container::getNodeId)); - // Ensure 2 containers allocated per node. + Map, List> outstandingSchedRequests = ((AMRMClientImpl)amClient).getOutstandingSchedRequests(); + // Check the outstanding SchedulingRequests + Assert.assertEquals(3, outstandingSchedRequests.size()); + Assert.assertEquals(1, outstandingSchedRequests.get(new HashSet<>(Collections.singletonList("foo"))).size()); + Assert.assertEquals(1, outstandingSchedRequests.get(new HashSet<>(Collections.singletonList("bar"))).size()); + Assert.assertEquals(0, outstandingSchedRequests.get(new HashSet()).size()); + // Each node should have a "foo" and a "bar" container. Assert.assertEquals(3, containersPerNode.entrySet().size()); HashSet srcTags = new HashSet<>(Arrays.asList("foo", "bar")); @@ -154,6 +162,7 @@ public float getProgress() { srcTags, x.getValue() .stream() + .filter(y -> !y.getAllocationTags().isEmpty()) .map(y -> y.getAllocationTags().iterator().next()) .collect(Collectors.toSet())) ); @@ -195,7 +204,7 @@ private static SchedulingRequest schedulingRequest(int numAllocations, return SchedulingRequest.newBuilder() .priority(Priority.newInstance(priority)) .allocationRequestId(allocReqId) - .allocationTags(new HashSet<>(Arrays.asList(tags))) + .allocationTags(tags == null ? new HashSet<>() : new HashSet<>(Arrays.asList(tags))) .executionType(ExecutionTypeRequest.newInstance(execType, true)) .resourceSizing( ResourceSizing.newInstance(numAllocations, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java index 34a9b34fc74..fe955723471 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java @@ -197,8 +197,6 @@ public static void removeFromOutstandingSchedulingRequests( return; } for (Container container : containers) { - if (container.getAllocationTags() != null - && !container.getAllocationTags().isEmpty()) { List schedReqs = outstandingSchedRequests.get(container.getAllocationTags()); if (schedReqs != null && !schedReqs.isEmpty()) { @@ -219,7 +217,6 @@ public static void removeFromOutstandingSchedulingRequests( } } } - } } } } \ No newline at end of file