schedulingRequests) {
if (schedulingRequests != null && !schedulingRequests.isEmpty()) {
- this.placementDispatcher.dispatch(
- new BatchedRequests(appAttemptId.getApplicationId(),
- schedulingRequests, 1));
+ this.placementDispatcher.dispatch(new BatchedRequests(iteratorType,
+ appAttemptId.getApplicationId(), schedulingRequests, 1));
}
}
@@ -329,11 +342,10 @@ private void addToRetryList(SchedulingResponse schedulerResponse,
}
}
if (!isAdded) {
- BatchedRequests br =
- new BatchedRequests(schedulerResponse.getApplicationId(),
- Collections.singleton(
- schedulerResponse.getSchedulingRequest()),
- placementAttempt + 1);
+ BatchedRequests br = new BatchedRequests(iteratorType,
+ schedulerResponse.getApplicationId(),
+ Collections.singleton(schedulerResponse.getSchedulingRequest()),
+ placementAttempt + 1);
reqsToRetry.add(br);
br.addToBlacklist(
schedulerResponse.getSchedulingRequest().getAllocationTags(),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
deleted file mode 100644
index 8d49801bd28..00000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
-
-import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutputCollector;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Sample Test algorithm. Assumes anti-affinity always
- * It also assumes the numAllocations in resource sizing is always = 1
- *
- * NOTE: This is just a sample implementation. Not be actually used
- */
-public class SamplePlacementAlgorithm implements ConstraintPlacementAlgorithm {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SamplePlacementAlgorithm.class);
-
- private AllocationTagsManager tagsManager;
- private PlacementConstraintManager constraintManager;
- private NodeCandidateSelector nodeSelector;
-
- @Override
- public void init(RMContext rmContext) {
- this.tagsManager = rmContext.getAllocationTagsManager();
- this.constraintManager = rmContext.getPlacementConstraintManager();
- this.nodeSelector =
- filter -> ((AbstractYarnScheduler)(rmContext)
- .getScheduler()).getNodes(filter);
- }
-
- @Override
- public void place(ConstraintPlacementAlgorithmInput input,
- ConstraintPlacementAlgorithmOutputCollector collector) {
- BatchedRequests requests = (BatchedRequests)input;
- ConstraintPlacementAlgorithmOutput resp =
- new ConstraintPlacementAlgorithmOutput(requests.getApplicationId());
- List allNodes = nodeSelector.selectNodes(null);
- Map> tagIndexedRequests = new HashMap<>();
- requests.getSchedulingRequests()
- .stream()
- .filter(r -> r.getAllocationTags() != null)
- .forEach(
- req -> req.getAllocationTags().forEach(
- tag -> tagIndexedRequests.computeIfAbsent(tag,
- k -> new ArrayList<>()).add(req))
- );
- for (Map.Entry> entry :
- tagIndexedRequests.entrySet()) {
- String tag = entry.getKey();
- PlacementConstraint constraint =
- constraintManager.getConstraint(requests.getApplicationId(),
- Collections.singleton(tag));
- if (constraint != null) {
- // Currently works only for simple anti-affinity
- // NODE scope target expressions
- SpecializedConstraintTransformer transformer =
- new SpecializedConstraintTransformer(constraint);
- PlacementConstraint transform = transformer.transform();
- TargetConstraint targetConstraint =
- (TargetConstraint) transform.getConstraintExpr();
- // Assume a single target expression tag;
- // The Sample Algorithm assumes a constraint will always be a simple
- // Target Constraint with a single entry in the target set.
- // As mentioned in the class javadoc - This algorithm should be
- // used mostly for testing and validating end-2-end workflow.
- String targetTag =
- targetConstraint.getTargetExpressions().iterator().next()
- .getTargetValues().iterator().next();
- // iterate over all nodes
- Iterator nodeIter = allNodes.iterator();
- List schedulingRequests = entry.getValue();
- Iterator reqIter = schedulingRequests.iterator();
- while (reqIter.hasNext()) {
- SchedulingRequest sReq = reqIter.next();
- int numAllocs = sReq.getResourceSizing().getNumAllocations();
- while (numAllocs > 0 && nodeIter.hasNext()) {
- SchedulerNode node = nodeIter.next();
- long nodeCardinality = 0;
- try {
- nodeCardinality = tagsManager.getNodeCardinality(
- node.getNodeID(), requests.getApplicationId(),
- targetTag);
- if (nodeCardinality == 0 &&
- !requests.getBlacklist(tag).contains(node.getNodeID())) {
- numAllocs--;
- sReq.getResourceSizing().setNumAllocations(numAllocs);
- PlacedSchedulingRequest placedReq =
- new PlacedSchedulingRequest(sReq);
- placedReq.setPlacementAttempt(requests.getPlacementAttempt());
- placedReq.getNodes().add(node);
- resp.getPlacedRequests().add(placedReq);
- }
- } catch (InvalidAllocationTagsQueryException e) {
- LOG.warn("Got exception from TagManager !", e);
- }
- }
- }
- }
- }
- // Add all requests whose numAllocations still > 0 to rejected list.
- requests.getSchedulingRequests().stream()
- .filter(sReq -> sReq.getResourceSizing().getNumAllocations() > 0)
- .forEach(rejReq -> resp.getRejectedRequests().add(rejReq));
- collector.collect(resp);
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
index 0ce16144d18..889066a5267 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
@@ -74,25 +74,20 @@ public void testAllocationTagsManagerSimpleCases()
*/
// 3 Containers from app1
- atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ atm.addContainer(NodeId.fromString("host1:123"), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ atm.addContainer(NodeId.fromString("host2:123"), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ atm.addContainer(NodeId.fromString("host1:123"), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ atm.addContainer(NodeId.fromString("host2:123"), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
// 1 Container from app2
- atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ atm.addContainer(NodeId.fromString("host2:123"), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
// Get Node Cardinality of app1 on node1, with tag "mapper"
@@ -170,24 +165,21 @@ public void testAllocationTagsManagerSimpleCases()
// Finish all containers:
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Expect all cardinality to be 0
// Get Cardinality of app1 on node1, with tag "mapper"
@@ -270,25 +262,22 @@ public void testAllocationTagsManagerRackMapping()
// 3 Containers from app1
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 2),
+ TestUtils.getMockContainerId(2, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(2, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
// 1 Container from app2
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Get Rack Cardinality of app1 on rack0, with tag "mapper"
Assert.assertEquals(1, atm.getRackCardinality("rack0",
@@ -325,45 +314,39 @@ public void testAllocationTagsManagerMemoryAfterCleanup()
// Add a bunch of containers
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Remove all these containers
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Check internal data structure
Assert.assertEquals(0,
@@ -375,6 +358,87 @@ public void testAllocationTagsManagerMemoryAfterCleanup()
}
@Test
+ public void testTempContainerAllocations()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Construct both TEMP and normal containers: Node1: TEMP container_1_1
+ * (mapper/reducer/app_1) container_1_2 (service/app_1)
+ *
+ * Node2: container_1_3 (reducer/app_1) TEMP container_2_1 (service/app_2)
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
+
+ // 3 Containers from app1
+ atm.addTempContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockContainerId(1, 2), ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("reducer"));
+
+ // 1 Container from app2
+ atm.addTempContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"));
+
+ // Expect tag mappings to be present including temp Tags
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("service"),
+ Long::sum));
+
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+
+ // Do a temp Tag cleanup on app2
+ atm.cleanTempContainers(TestUtils.getMockApplicationId(2));
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+ // Expect app1 to be unaffected
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+ // Do a cleanup on app1 as well
+ atm.cleanTempContainers(TestUtils.getMockApplicationId(1));
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+
+ // Non temp-tags should be unaffected
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("service"),
+ Long::sum));
+
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+
+ // Expect app2 with no containers, and app1 with 2 containers across 2 nodes
+ Assert.assertEquals(2,
+ atm.getPerAppNodeMappings().get(TestUtils.getMockApplicationId(1))
+ .getTypeToTagsWithCount().size());
+
+ Assert.assertNull(
+ atm.getPerAppNodeMappings().get(TestUtils.getMockApplicationId(2)));
+ }
+
+ @Test
public void testQueryCardinalityWithIllegalParameters()
throws InvalidAllocationTagsQueryException {
/**
@@ -385,24 +449,21 @@ public void testQueryCardinalityWithIllegalParameters()
// Add a bunch of containers
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// No node-id
boolean caughtException = false;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java
new file mode 100644
index 00000000000..b7f9e70a224
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java
@@ -0,0 +1,61 @@
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.TestPlacementProcessor.schedulingRequest;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.BatchedRequests;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestBatchedRequestsIterators {
+
+ @Test
+ public void testSerialIterator() throws Exception {
+ List schedulingRequestList =
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"));
+
+ BatchedRequests batchedRequests = new BatchedRequests(
+ BatchedRequests.IteratorType.SERIAL, null, schedulingRequestList, 1);
+
+ Iterator requestIterator = batchedRequests.iterator();
+ long prevAllocId = 0;
+ while (requestIterator.hasNext()) {
+ SchedulingRequest request = requestIterator.next();
+ Assert.assertTrue(request.getAllocationRequestId() > prevAllocId);
+ prevAllocId = request.getAllocationRequestId();
+ }
+ }
+
+ @Test
+ public void testPopularTagsIterator() throws Exception {
+ List schedulingRequestList =
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "pri", "foo"),
+ schedulingRequest(1, 2, 1, 512, "bar"),
+ schedulingRequest(1, 3, 1, 512, "foo", "pri"),
+ schedulingRequest(1, 4, 1, 512, "test"),
+ schedulingRequest(1, 5, 1, 512, "pri", "bar"));
+
+ BatchedRequests batchedRequests =
+ new BatchedRequests(BatchedRequests.IteratorType.POPULAR_TAGS, null,
+ schedulingRequestList, 1);
+
+ Iterator requestIterator = batchedRequests.iterator();
+ long recCcount = 0;
+ while (requestIterator.hasNext()) {
+ SchedulingRequest request = requestIterator.next();
+ if (recCcount < 3)
+ Assert.assertTrue(request.getAllocationTags().contains("pri"));
+ else
+ Assert.assertTrue(request.getAllocationTags().contains("bar")
+ || request.getAllocationTags().contains("test"));
+ recCcount++;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
index db8ae153824..87dd5b71117 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
@@ -373,13 +373,13 @@ public void testPlacementRejection() throws Exception {
rej.getReason());
}
- private static SchedulingRequest schedulingRequest(
+ protected static SchedulingRequest schedulingRequest(
int priority, long allocReqId, int cores, int mem, String... tags) {
return schedulingRequest(priority, allocReqId, cores, mem,
ExecutionType.GUARANTEED, tags);
}
- private static SchedulingRequest schedulingRequest(
+ protected static SchedulingRequest schedulingRequest(
int priority, long allocReqId, int cores, int mem,
ExecutionType execType, String... tags) {
return SchedulingRequest.newBuilder()