diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java index 5a61b94..496321d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java @@ -97,8 +97,8 @@ private boolean computeAllocation(ReservationId reservationId, String user, long curDeadline = deadline; long oldDeadline = -1; - Map allocations = - new HashMap(); + Map allocations = + new HashMap<>(); RLESparseResourceAllocation tempAssigned = new RLESparseResourceAllocation(plan.getResourceCalculator(), plan.getMinimumAllocation()); @@ -108,6 +108,8 @@ private boolean computeAllocation(ReservationId reservationId, String user, ReservationRequestInterpreter type = contract.getReservationRequests() .getInterpreter(); + boolean hasGang = false; + // Iterate the stages in backward from deadline for (ListIterator li = stages.listIterator(stages.size()); li.hasPrevious();) { @@ -117,8 +119,10 @@ private boolean computeAllocation(ReservationId reservationId, String user, // validate the RR respect basic constraints validateInput(plan, currentReservationStage, totalCapacity); + hasGang |= currentReservationStage.getConcurrency() > 1; + // run allocation for a single stage - Map curAlloc = + Map curAlloc = placeSingleStage(plan, tempAssigned, currentReservationStage, earliestStart, curDeadline, oldReservation, totalCapacity); @@ -178,8 +182,7 @@ private boolean computeAllocation(ReservationId reservationId, String user, // create reservation with above allocations if not null/empty - ReservationRequest ZERO_RES = - ReservationRequest.newInstance(Resource.newInstance(0, 0), 0); + Resource ZERO_RES = Resource.newInstance(0, 0); long firstStartTime = findEarliestTime(allocations.keySet()); @@ -200,7 +203,7 @@ private boolean computeAllocation(ReservationId reservationId, String user, new InMemoryReservationAllocation(reservationId, contract, user, plan.getQueueName(), firstStartTime, findLatestTime(allocations.keySet()), allocations, - plan.getResourceCalculator(), plan.getMinimumAllocation()); + plan.getResourceCalculator(), plan.getMinimumAllocation(), hasGang); if (oldReservation != null) { return plan.updateReservation(capReservation); } else { @@ -242,13 +245,13 @@ private void validateInput(Plan plan, ReservationRequest rr, * previous instant in time until the time-window is exhausted or we placed * all the user request. */ - private Map placeSingleStage( + private Map placeSingleStage( Plan plan, RLESparseResourceAllocation tempAssigned, ReservationRequest rr, long earliestStart, long curDeadline, ReservationAllocation oldResAllocation, final Resource totalCapacity) { - Map allocationRequests = - new HashMap(); + Map allocationRequests = + new HashMap<>(); // compute the gang as a resource and get the duration Resource gang = Resources.multiply(rr.getCapability(), rr.getConcurrency()); @@ -322,7 +325,7 @@ private void validateInput(Plan plan, ReservationRequest rr, ReservationInterval reservationInt = new ReservationInterval(curDeadline - dur, curDeadline); - ReservationRequest reservationRes = + ReservationRequest reservationRequest = ReservationRequest.newInstance(rr.getCapability(), rr.getConcurrency() * maxGang, rr.getConcurrency(), rr.getDuration()); @@ -331,6 +334,8 @@ private void validateInput(Plan plan, ReservationRequest rr, // placing other ReservationRequest within the same // ReservationDefinition, // and we must avoid double-counting the available resources + final Resource reservationRes = ReservationUtil.toResource( + reservationRequest); tempAssigned.addInterval(reservationInt, reservationRes); allocationRequests.put(reservationInt, reservationRes); @@ -350,7 +355,7 @@ private void validateInput(Plan plan, ReservationRequest rr, // If we are here is becasue we did not manage to satisfy this request. // So we need to remove unwanted side-effect from tempAssigned (needed // for ANY). - for (Map.Entry tempAllocation : + for (Map.Entry tempAllocation : allocationRequests.entrySet()) { tempAssigned.removeInterval(tempAllocation.getKey(), tempAllocation.getValue()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java index ce2e7d7..50d66cf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java @@ -31,7 +31,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -110,7 +109,7 @@ public QueueMetrics getQueueMetrics() { private void incrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); - Map allocationRequests = + Map allocationRequests = reservation.getAllocationRequests(); // check if we have encountered the user earlier and if not add an entry String user = reservation.getUser(); @@ -119,7 +118,7 @@ private void incrementAllocation(ReservationAllocation reservation) { resAlloc = new RLESparseResourceAllocation(resCalc, minAlloc); userResourceAlloc.put(user, resAlloc); } - for (Map.Entry r : allocationRequests + for (Map.Entry r : allocationRequests .entrySet()) { resAlloc.addInterval(r.getKey(), r.getValue()); rleSparseVector.addInterval(r.getKey(), r.getValue()); @@ -128,11 +127,11 @@ private void incrementAllocation(ReservationAllocation reservation) { private void decrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); - Map allocationRequests = + Map allocationRequests = reservation.getAllocationRequests(); String user = reservation.getUser(); RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user); - for (Map.Entry r : allocationRequests + for (Map.Entry r : allocationRequests .entrySet()) { resAlloc.removeInterval(r.getKey(), r.getValue()); rleSparseVector.removeInterval(r.getKey(), r.getValue()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java index fc8407b..a4dd23b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryReservationAllocation.java @@ -22,7 +22,6 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -40,7 +39,7 @@ private final ReservationDefinition contract; private final long startTime; private final long endTime; - private final Map allocationRequests; + private final Map allocationRequests; private boolean hasGang = false; private long acceptedAt = -1; @@ -49,22 +48,29 @@ InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, - Map allocationRequests, + Map allocations, ResourceCalculator calculator, Resource minAlloc) { + this(reservationID, contract, user, planName, startTime, endTime, + allocations, calculator, minAlloc, false); + } + + InMemoryReservationAllocation(ReservationId reservationID, + ReservationDefinition contract, String user, String planName, + long startTime, long endTime, + Map allocations, + ResourceCalculator calculator, Resource minAlloc, boolean hasGang) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; - this.allocationRequests = allocationRequests; + this.allocationRequests = allocations; this.planName = planName; + this.hasGang = hasGang; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); - for (Map.Entry r : allocationRequests + for (Map.Entry r : allocations .entrySet()) { resourcesOverTime.addInterval(r.getKey(), r.getValue()); - if (r.getValue().getConcurrency() > 1) { - hasGang = true; - } } } @@ -89,7 +95,7 @@ public long getEndTime() { } @Override - public Map getAllocationRequests() { + public Map getAllocationRequests() { return Collections.unmodifiableMap(allocationRequests); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java index 3f6f405..2957cc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.StringWriter; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.NavigableMap; @@ -31,9 +30,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -80,14 +77,11 @@ private boolean isSameAsNext(Long key, Resource capacity) { * * @param reservationInterval the interval for which the resource is to be * added - * @param capacity the resource to be added + * @param totCap the resource to be added * @return true if addition is successful, false otherwise */ public boolean addInterval(ReservationInterval reservationInterval, - ReservationRequest capacity) { - Resource totCap = - Resources.multiply(capacity.getCapability(), - (float) capacity.getNumContainers()); + Resource totCap) { if (totCap.equals(ZERO_RESOURCE)) { return true; } @@ -143,44 +137,15 @@ public boolean addInterval(ReservationInterval reservationInterval, } /** - * Add multiple resources for the specified interval - * - * @param reservationInterval the interval for which the resource is to be - * added - * @param ReservationRequests the resources to be added - * @param clusterResource the total resources in the cluster - * @return true if addition is successful, false otherwise - */ - public boolean addCompositeInterval(ReservationInterval reservationInterval, - List ReservationRequests, Resource clusterResource) { - ReservationRequest aggregateReservationRequest = - Records.newRecord(ReservationRequest.class); - Resource capacity = Resource.newInstance(0, 0); - for (ReservationRequest ReservationRequest : ReservationRequests) { - Resources.addTo(capacity, Resources.multiply( - ReservationRequest.getCapability(), - ReservationRequest.getNumContainers())); - } - aggregateReservationRequest.setNumContainers((int) Math.ceil(Resources - .divide(resourceCalculator, clusterResource, capacity, minAlloc))); - aggregateReservationRequest.setCapability(minAlloc); - - return addInterval(reservationInterval, aggregateReservationRequest); - } - - /** * Removes a resource for the specified interval * * @param reservationInterval the interval for which the resource is to be * removed - * @param capacity the resource to be removed + * @param totCap the resource to be removed * @return true if removal is successful, false otherwise */ public boolean removeInterval(ReservationInterval reservationInterval, - ReservationRequest capacity) { - Resource totCap = - Resources.multiply(capacity.getCapability(), - (float) capacity.getNumContainers()); + Resource totCap) { if (totCap.equals(ZERO_RESOURCE)) { return true; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java index 89c0e55..0d3c692 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationAllocation.java @@ -22,7 +22,6 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.Resource; /** @@ -71,7 +70,7 @@ * @return the allocationRequests the map of resources requested against the * time interval for which they were */ - public Map getAllocationRequests(); + public Map getAllocationRequests(); /** * Return a string identifying the plan to which the reservation belongs diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationUtil.java new file mode 100644 index 0000000..47b71f9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationUtil.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.reservation; + +import org.apache.hadoop.yarn.api.records.ReservationRequest; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.util.HashMap; +import java.util.Map; + +final class ReservationUtil { + + private ReservationUtil() { + // not called + } + + public static Resource toResource(ReservationRequest request) { + Resource resource = Resources.multiply(request.getCapability(), + (float) request.getNumContainers()); + return resource; + } + + public static Map toResources( + Map allocations) { + Map resources = new HashMap<>(); + for (Map.Entry entry : + allocations.entrySet()) { + resources.put(entry.getKey(), + toResource(entry.getValue())); + } + return resources; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index bfaf06b..1f78a50 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -378,14 +378,14 @@ public static ReservationDefinition generateBigRR(Random rand, long i) { return rr; } - public static Map generateAllocation( + public static Map generateAllocation( long startTime, long step, int[] alloc) { - Map req = - new TreeMap(); + Map req = + new TreeMap<>(); for (int i = 0; i < alloc.length; i++) { req.put(new ReservationInterval(startTime + i * step, startTime + (i + 1) - * step), ReservationRequest.newInstance( - Resource.newInstance(1024, 1), alloc[i])); + * step), ReservationUtil.toResource(ReservationRequest.newInstance( + Resource.newInstance(1024, 1), alloc[i]))); } return req; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java index 61561e9..88dfbe3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java @@ -198,12 +198,14 @@ public void testInstFailBySum() throws IOException, PlanningException { @Test(expected = PlanningQuotaException.class) public void testFailAvg() throws IOException, PlanningException { // generate an allocation which violates the 25% average single-shot - Map req = - new TreeMap(); + Map req = + new TreeMap<>(); long win = timeWindow / 2 + 100; int cont = (int) Math.ceil(0.5 * totCont); req.put(new ReservationInterval(initTime, initTime + win), - ReservationRequest.newInstance(Resource.newInstance(1024, 1), cont)); + ReservationUtil.toResource( + ReservationRequest.newInstance(Resource.newInstance(1024, 1), + cont))); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( @@ -214,12 +216,13 @@ public void testFailAvg() throws IOException, PlanningException { @Test public void testFailAvgBySum() throws IOException, PlanningException { // generate an allocation which violates the 25% average by sum - Map req = - new TreeMap(); + Map req = + new TreeMap<>(); long win = 86400000 / 4 + 1; int cont = (int) Math.ceil(0.5 * totCont); req.put(new ReservationInterval(initTime, initTime + win), - ReservationRequest.newInstance(Resource.newInstance(1024, 1), cont)); + ReservationUtil.toResource(ReservationRequest.newInstance(Resource + .newInstance(1024, 1), cont))); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), null, "u1", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java index b8cf6c5..de94dcd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java @@ -516,7 +516,7 @@ private void prepareBasicPlan() throws PlanningException { .generateAllocation(0, step, f), res, minAlloc))); int[] f2 = { 5, 5, 5, 5, 5, 5, 5 }; - Map alloc = + Map alloc = ReservationSystemTestUtil.generateAllocation(5000, step, f2); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java index 91c1962..fc3801e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java @@ -95,11 +95,13 @@ public void testAddReservation() { ReservationSystemTestUtil.getNewReservationId(); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; - Map allocations = + Map allocs = generateAllocation(start, alloc, false); ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, - alloc.length, allocations.values()); + alloc.length, allocs.values()); + Map allocations = + ReservationUtil.toResources(allocs); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length, allocations, resCalc, minAlloc); @@ -132,9 +134,11 @@ public void testAddEmptyReservation() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); + Map allocs = ReservationUtil.toResources + (allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length, allocations, resCalc, minAlloc); + start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); @@ -158,9 +162,11 @@ public void testAddReservationAlreadyExists() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); + Map allocs = ReservationUtil.toResources + (allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length, allocations, resCalc, minAlloc); + start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); @@ -202,9 +208,11 @@ public void testUpdateReservation() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); + Map allocs = ReservationUtil.toResources + (allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length, allocations, resCalc, minAlloc); + start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); @@ -226,9 +234,12 @@ public void testUpdateReservation() { rDef = createSimpleReservationDefinition(start, start + updatedAlloc.length, updatedAlloc.length, allocations.values()); + Map updatedAllocs = + ReservationUtil.toResources(allocations); rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + updatedAlloc.length, allocations, resCalc, minAlloc); + start, start + updatedAlloc.length, updatedAllocs, resCalc, + minAlloc); try { plan.updateReservation(rAllocation); } catch (PlanningException e) { @@ -260,9 +271,11 @@ public void testUpdateNonExistingReservation() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); + Map allocs = + ReservationUtil.toResources(allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length, allocations, resCalc, minAlloc); + start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.updateReservation(rAllocation); @@ -290,9 +303,11 @@ public void testDeleteReservation() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); + Map allocs = + ReservationUtil.toResources(allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length, allocations, resCalc, minAlloc); + start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); @@ -359,9 +374,11 @@ public void testArchiveCompletedReservations() { ReservationDefinition rDef1 = createSimpleReservationDefinition(start, start + alloc1.length, alloc1.length, allocations1.values()); + Map allocs1 = + ReservationUtil.toResources(allocations1); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID1, rDef1, user, - planName, start, start + alloc1.length, allocations1, resCalc, + planName, start, start + alloc1.length, allocs1, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID1)); try { @@ -388,9 +405,11 @@ public void testArchiveCompletedReservations() { ReservationDefinition rDef2 = createSimpleReservationDefinition(start, start + alloc2.length, alloc2.length, allocations2.values()); + Map allocs2 = + ReservationUtil.toResources(allocations2); rAllocation = new InMemoryReservationAllocation(reservationID2, rDef2, user, - planName, start, start + alloc2.length, allocations2, resCalc, + planName, start, start + alloc2.length, allocs2, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID2)); try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java index 76f39dc..11ae4da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryReservationAllocation.java @@ -69,11 +69,12 @@ public void testBlocks() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); - Map allocations = + Map allocations = generateAllocation(start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length + 1, allocations, resCalc, minAlloc); + start, start + alloc.length + 1, allocations, resCalc, minAlloc, + false); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { @@ -91,11 +92,12 @@ public void testSteps() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); - Map allocations = + Map allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length + 1, allocations, resCalc, minAlloc); + start, start + alloc.length + 1, allocations, resCalc, minAlloc, + false); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { @@ -114,11 +116,12 @@ public void testSkyline() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); - Map allocations = + Map allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length + 1, allocations, resCalc, minAlloc); + start, start + alloc.length + 1, allocations, resCalc, minAlloc, + false); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { @@ -137,11 +140,12 @@ public void testZeroAlloaction() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); - Map allocations = - new HashMap(); + Map allocations = + new HashMap<>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length + 1, allocations, resCalc, minAlloc); + start, start + alloc.length + 1, allocations, resCalc, minAlloc, + false); doAssertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc); Assert.assertFalse(rAllocation.containsGangs()); @@ -156,11 +160,13 @@ public void testGangAlloaction() { ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); - Map allocations = - generateAllocation(start, alloc, false, true); + boolean isGang = true; + Map allocations = + generateAllocation(start, alloc, false, isGang); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, - start, start + alloc.length + 1, allocations, resCalc, minAlloc); + start, start + alloc.length + 1, allocations, resCalc, minAlloc, + isGang); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertTrue(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { @@ -171,11 +177,12 @@ public void testGangAlloaction() { private void doAssertions(ReservationAllocation rAllocation, ReservationId reservationID, ReservationDefinition rDef, - Map allocations, int start, + Map allocations, int start, int[] alloc) { Assert.assertEquals(reservationID, rAllocation.getReservationId()); Assert.assertEquals(rDef, rAllocation.getReservationDefinition()); - Assert.assertEquals(allocations, rAllocation.getAllocationRequests()); + Assert.assertEquals(allocations, + rAllocation.getAllocationRequests()); Assert.assertEquals(user, rAllocation.getUser()); Assert.assertEquals(planName, rAllocation.getPlanName()); Assert.assertEquals(start, rAllocation.getStartTime()); @@ -198,10 +205,10 @@ private ReservationDefinition createSimpleReservationDefinition(long arrival, return rDef; } - private Map generateAllocation( + private Map generateAllocation( int startTime, int[] alloc, boolean isStep, boolean isGang) { - Map req = - new HashMap(); + Map req = + new HashMap<>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { @@ -215,7 +222,8 @@ private ReservationDefinition createSimpleReservationDefinition(long arrival, if (isGang) { rr.setConcurrency(numContainers); } - req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr); + req.put(new ReservationInterval(startTime + i, startTime + i + 1), + ReservationUtil.toResource(rr)); } return req; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java index c7301c7..622fcc2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java @@ -49,7 +49,8 @@ public void testBlocks() { Set> inputs = generateAllocation(start, alloc, false).entrySet(); for (Entry ip : inputs) { - rleSparseVector.addInterval(ip.getKey(), ip.getValue()); + rleSparseVector.addInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); Assert.assertFalse(rleSparseVector.isEmpty()); @@ -64,7 +65,8 @@ public void testBlocks() { Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 2)); for (Entry ip : inputs) { - rleSparseVector.removeInterval(ip.getKey(), ip.getValue()); + rleSparseVector.removeInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); for (int i = 0; i < alloc.length; i++) { @@ -86,7 +88,8 @@ public void testSteps() { Set> inputs = generateAllocation(start, alloc, true).entrySet(); for (Entry ip : inputs) { - rleSparseVector.addInterval(ip.getKey(), ip.getValue()); + rleSparseVector.addInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); Assert.assertFalse(rleSparseVector.isEmpty()); @@ -102,7 +105,8 @@ public void testSteps() { Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 2)); for (Entry ip : inputs) { - rleSparseVector.removeInterval(ip.getKey(), ip.getValue()); + rleSparseVector.removeInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); for (int i = 0; i < alloc.length; i++) { @@ -124,7 +128,8 @@ public void testSkyline() { Set> inputs = generateAllocation(start, alloc, true).entrySet(); for (Entry ip : inputs) { - rleSparseVector.addInterval(ip.getKey(), ip.getValue()); + rleSparseVector.addInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); Assert.assertFalse(rleSparseVector.isEmpty()); @@ -140,7 +145,8 @@ public void testSkyline() { Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 2)); for (Entry ip : inputs) { - rleSparseVector.removeInterval(ip.getKey(), ip.getValue()); + rleSparseVector.removeInterval(ip.getKey(), + ReservationUtil.toResource(ip.getValue())); } LOG.info(rleSparseVector.toString()); for (int i = 0; i < alloc.length; i++) { @@ -157,7 +163,7 @@ public void testZeroAlloaction() { RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc); rleSparseVector.addInterval(new ReservationInterval(0, Long.MAX_VALUE), - ReservationRequest.newInstance(Resource.newInstance(0, 0), (0))); + Resource.newInstance(0, 0)); LOG.info(rleSparseVector.toString()); Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(new Random().nextLong())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java index 1ca9f2e..694f44c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java @@ -146,14 +146,15 @@ public void testReplanningPlanCapacityLoss() throws PlanningException { } } - private Map generateAllocation( + private Map generateAllocation( int startTime, int[] alloc) { - Map req = - new TreeMap(); + Map req = + new TreeMap<>(); for (int i = 0; i < alloc.length; i++) { req.put(new ReservationInterval(startTime + i, startTime + i + 1), - ReservationRequest.newInstance(Resource.newInstance(1024, 1), - alloc[i])); + ReservationUtil.toResource( + ReservationRequest.newInstance(Resource.newInstance(1024, 1), + alloc[i]))); } return req; }