diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index bdf60bd9a5b..457fe47ccbe 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -130,6 +130,11 @@ public boolean fitsIn(Resource cluster,
@Override
public boolean isAnyMajorResourceZero(Resource resource) {
- return resource.getMemorySize() == 0f;
+ return resource.getMemorySize() == 0;
+ }
+
+ @Override
+ public boolean isNonZero(Resource resource) {
+ return resource.getMemorySize() > 0;
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 7697e1dfc33..2b615474aee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -242,6 +242,11 @@ public boolean fitsIn(Resource cluster,
@Override
public boolean isAnyMajorResourceZero(Resource resource) {
- return resource.getMemorySize() == 0f || resource.getVirtualCores() == 0;
+ return resource.getMemorySize() == 0 || resource.getVirtualCores() == 0;
+ }
+
+ @Override
+ public boolean isNonZero(Resource resource) {
+ return (resource.getMemorySize() > 0) || (resource.getVirtualCores() > 0);
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 398dac50fa5..14390206d0a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -236,4 +236,13 @@ public abstract boolean fitsIn(Resource cluster,
* @return returns true if any resource is zero.
*/
public abstract boolean isAnyMajorResourceZero(Resource resource);
+
+ /**
+ * Return true if any major resource value in {@code resource} is greater
+ * than 0.
+ *
+ * @param resource the resource to test
+ * @return whether the resource has non-zero major resource values
+ */
+ public abstract boolean isNonZero(Resource resource);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index a1d14fdce73..d41892e4c9e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -146,7 +146,7 @@ public static Resource none() {
public static boolean isNone(Resource other) {
return NONE.equals(other);
}
-
+
public static Resource unbounded() {
return UNBOUNDED;
}
@@ -176,8 +176,9 @@ public static Resource subtract(Resource lhs, Resource rhs) {
}
/**
- * Subtract rhs from lhs and reset any negative
- * values to zero.
+ * Subtract {@code rhs} from {@code lhs} and reset any negative values to
+ * zero. This call will modify {@code lhs}.
+ *
* @param lhs {@link Resource} to subtract from
* @param rhs {@link Resource} to subtract
* @return the value of lhs after subtraction
@@ -193,6 +194,19 @@ public static Resource subtractFromNonNegative(Resource lhs, Resource rhs) {
return lhs;
}
+ /**
+ * Subtract {@code rhs} from {@code lhs} and reset any negative values to
+ * zero. This call will operate on a copy of {@code lhs}, leaving {@code lhs}
+ * unmodified.
+ *
+ * @param lhs {@link Resource} to subtract from
+ * @param rhs {@link Resource} to subtract
+ * @return the value of lhs after subtraction
+ */
+ public static Resource subtractNonNegative(Resource lhs, Resource rhs) {
+ return subtractFromNonNegative(clone(lhs), rhs);
+ }
+
public static Resource negate(Resource resource) {
return subtract(NONE, resource);
}
@@ -355,4 +369,17 @@ public static boolean isAnyMajorResourceZero(ResourceCalculator rc,
Resource resource) {
return rc.isAnyMajorResourceZero(resource);
}
+
+
+ /**
+ * Return true if any major resource value in {@code resource} is greater
+ * than 0 according to {@code rc}.
+ *
+ * @param rc the resource calculator to use
+ * @param res the resource to test
+ * @return whether the resource has non-zero major resource values
+ */
+ public static boolean isNonZero(ResourceCalculator rc, Resource res) {
+ return rc.isNonZero(res);
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index b123b0520d4..7ad225bd1c7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -23,6 +23,8 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.junit.Assert;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -229,4 +231,43 @@ public void testNormalize() {
Assert.assertEquals(2, result.getVirtualCores());
}
}
+
+ @Test
+ public void testIsNonZero() {
+ Resource r = Resource.newInstance(0L, 0);
+
+ assertFalse("isNonZero() should not have returned true for a resource with "
+ + "0MB memory and 0 vcores",
+ Resources.isNonZero(resourceCalculator, r));
+
+ r = Resource.newInstance(0L, 1);
+
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ assertFalse("isNonZero() should not have returned true for a resource "
+ + "with 0MB memory and 1 vcore",
+ Resources.isNonZero(resourceCalculator, r));
+ } else {
+ assertTrue("isNonZero() should have returned true for a resource with "
+ + "0MB memory and 1 vcore",
+ Resources.isNonZero(resourceCalculator, r));
+ }
+
+ r = Resource.newInstance(1L, 0);
+
+ assertTrue("isNonZero() should have returned true for a resource with "
+ + "1MB memory and 0 vcores",
+ Resources.isNonZero(resourceCalculator, r));
+
+ r = Resource.newInstance(1L, 1);
+
+ assertTrue("isNonZero() should have returned true for a resource with "
+ + "1MB memory and 1 vcore",
+ Resources.isNonZero(resourceCalculator, r));
+
+ r = Resource.newInstance(Long.MAX_VALUE, Integer.MAX_VALUE);
+
+ assertTrue("isNonZero() should have returned true for a resource with "
+ + "maximum memory and vcores",
+ Resources.isNonZero(resourceCalculator, r));
+ }
}
\ No newline at end of file
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f9a72191a67..2c538385239 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1190,8 +1190,7 @@ public void recordContainerAllocationTime(long value) {
}
@Private
- public boolean hasPendingResourceRequest(ResourceCalculator rc,
- String nodePartition, Resource cluster,
+ public boolean hasPendingResourceRequest(String nodePartition,
SchedulingMode schedulingMode) {
// We need to consider unconfirmed allocations
if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
@@ -1204,16 +1203,12 @@ public boolean hasPendingResourceRequest(ResourceCalculator rc,
// To avoid too many allocation-proposals rejected for non-default
// partition allocation
if (StringUtils.equals(nodePartition, RMNodeLabelsManager.NO_LABEL)) {
- pending = Resources.subtract(pending, Resources
+ pending = Resources.subtractNonNegative(pending, Resources
.createResource(unconfirmedAllocatedMem.get(),
unconfirmedAllocatedVcores.get()));
}
- if (Resources.greaterThan(rc, cluster, pending, Resources.none())) {
- return true;
- }
-
- return false;
+ return !Resources.isNone(pending);
}
/*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index f753d31fdbf..b3eca936fe3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -838,8 +838,8 @@ public CSAssignment assignContainers(Resource clusterResource,
if (reservedContainer == null) {
// Check if application needs more resource, skip if it doesn't need more.
- if (!application.hasPendingResourceRequest(rc,
- ps.getPartition(), clusterResource, schedulingMode)) {
+ if (!application.hasPendingResourceRequest(ps.getPartition(),
+ schedulingMode)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId()
+ ", because it doesn't need more resource, schedulingMode="
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 71e6f7fd7df..7317e3becd9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -289,7 +289,7 @@ Resource getMinResources(String queue) {
}
/**
- * Get the maximum resource allocation for the given queue. If the max in not
+ * Get the maximum resource allocation for the given queue. If the max is not
* set, return the larger of the min and the default max.
*
* @param queue the target queue's name
@@ -300,10 +300,9 @@ Resource getMaxResources(String queue) {
Resource maxQueueResource = maxQueueResources.get(queue);
if (maxQueueResource == null) {
Resource minQueueResource = minQueueResources.get(queue);
- if (minQueueResource != null &&
- Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(),
- minQueueResource, queueMaxResourcesDefault)) {
- return minQueueResource;
+ if (minQueueResource != null) {
+ return Resources.componentwiseMax(minQueueResource,
+ queueMaxResourcesDefault);
} else {
return queueMaxResourcesDefault;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index fe57d1b9c3a..174bb63a47f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -1045,8 +1045,7 @@ private boolean hasContainerForNode(SchedulerRequestKey key,
(!hasRequestForRack || appSchedulingInfo.canDelayTo(key,
node.getRackName()) || (hasRequestForNode)) &&
// The requested container must be able to fit on the node:
- Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null,
- resource,
+ Resources.fitsIn(resource,
node.getRMNode().getTotalCapability()))) {
ret = false;
} else if (!getQueue().fitsInMaxShare(resource)) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index b911a1ae71a..56491f73fb5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -563,13 +563,14 @@ public void setWeights(float weight) {
*/
private Resource minShareStarvation() {
// If demand < minshare, we should use demand to determine starvation
- Resource desiredShare = Resources.min(policy.getResourceCalculator(),
- scheduler.getClusterResource(), getMinShare(), getDemand());
+ Resource starvation =
+ Resources.componentwiseMin(getMinShare(), getDemand());
- Resource starvation = Resources.subtract(desiredShare, getResourceUsage());
- boolean starved = !Resources.isNone(starvation);
+ Resources.subtractFromNonNegative(starvation, getResourceUsage());
+ boolean starved = !Resources.isNone(starvation);
long now = scheduler.getClock().getTime();
+
if (!starved) {
// Record that the queue is not starved
setLastTimeAtMinShare(now);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 0ef90a1d72f..541f7e7a123 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -82,17 +82,18 @@ public String getName() {
private static class FairShareComparator implements Comparator,
Serializable {
private static final long serialVersionUID = 5564969375856699313L;
- private static final Resource ONE = Resources.createResource(1);
@Override
public int compare(Schedulable s1, Schedulable s2) {
int res = compareDemand(s1, s2);
- // Pre-compute resource usages to avoid duplicate calculation
- Resource resourceUsage1 = s1.getResourceUsage();
- Resource resourceUsage2 = s2.getResourceUsage();
+ // Share resource usages to avoid duplicate calculation
+ Resource resourceUsage1 = null;
+ Resource resourceUsage2 = null;
if (res == 0) {
+ resourceUsage1 = s1.getResourceUsage();
+ resourceUsage2 = s2.getResourceUsage();
res = compareMinShareUsage(s1, s2, resourceUsage1, resourceUsage2);
}
@@ -115,41 +116,44 @@ public int compare(Schedulable s1, Schedulable s2) {
private int compareDemand(Schedulable s1, Schedulable s2) {
int res = 0;
- Resource demand1 = s1.getDemand();
- Resource demand2 = s2.getDemand();
- if (demand1.equals(Resources.none()) && Resources.greaterThan(
- RESOURCE_CALCULATOR, null, demand2, Resources.none())) {
+ long demand1 = s1.getDemand().getMemorySize();
+ long demand2 = s2.getDemand().getMemorySize();
+
+ if ((demand1 == 0) && (demand2 > 0)) {
res = 1;
- } else if (demand2.equals(Resources.none()) && Resources.greaterThan(
- RESOURCE_CALCULATOR, null, demand1, Resources.none())) {
+ } else if ((demand2 == 0) && (demand1 > 0)) {
res = -1;
}
+
return res;
}
private int compareMinShareUsage(Schedulable s1, Schedulable s2,
Resource resourceUsage1, Resource resourceUsage2) {
int res;
- Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
- s1.getMinShare(), s1.getDemand());
- Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
- s2.getMinShare(), s2.getDemand());
- boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
- resourceUsage1, minShare1);
- boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
- resourceUsage2, minShare2);
+ long minShare1 = Math.min(s1.getMinShare().getMemorySize(),
+ s1.getDemand().getMemorySize());
+ long minShare2 = Math.min(s2.getMinShare().getMemorySize(),
+ s2.getDemand().getMemorySize());
+ boolean s1Needy = resourceUsage1.getMemorySize() < minShare1;
+ boolean s2Needy = resourceUsage2.getMemorySize() < minShare2;
if (s1Needy && !s2Needy) {
res = -1;
} else if (s2Needy && !s1Needy) {
res = 1;
} else if (s1Needy && s2Needy) {
- double minShareRatio1 = (double) resourceUsage1.getMemorySize() /
- Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE)
- .getMemorySize();
- double minShareRatio2 = (double) resourceUsage2.getMemorySize() /
- Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE)
- .getMemorySize();
+ double minShareRatio1 = (double) resourceUsage1.getMemorySize();
+ double minShareRatio2 = (double) resourceUsage2.getMemorySize();
+
+ if (minShare1 > 1) {
+ minShareRatio1 /= minShare1;
+ }
+
+ if (minShare2 > 1) {
+ minShareRatio2 /= minShare2;
+ }
+
res = (int) Math.signum(minShareRatio1 - minShareRatio2);
} else {
res = 0;
@@ -168,21 +172,20 @@ private int compareFairShareUsage(Schedulable s1, Schedulable s2,
double weight2 = s2.getWeights().getWeight(ResourceType.MEMORY);
double useToWeightRatio1;
double useToWeightRatio2;
+
if (weight1 > 0.0 && weight2 > 0.0) {
useToWeightRatio1 = resourceUsage1.getMemorySize() / weight1;
useToWeightRatio2 = resourceUsage2.getMemorySize() / weight2;
- } else { // Either weight1 or weight2 equals to 0
- if (weight1 == weight2) {
- // If they have same weight, just compare usage
- useToWeightRatio1 = resourceUsage1.getMemorySize();
- useToWeightRatio2 = resourceUsage2.getMemorySize();
- } else {
- // By setting useToWeightRatios to negative weights, we give the
- // zero-weight one less priority, so the non-zero weight one will
- // be given slots.
- useToWeightRatio1 = -weight1;
- useToWeightRatio2 = -weight2;
- }
+ } else if (weight1 == weight2) { // Either weight1 or weight2 equals to 0
+ // If they have same weight, just compare usage
+ useToWeightRatio1 = resourceUsage1.getMemorySize();
+ useToWeightRatio2 = resourceUsage2.getMemorySize();
+ } else {
+ // By setting useToWeightRatios to negative weights, we give the
+ // zero-weight one less priority, so the non-zero weight one will
+ // be given slots.
+ useToWeightRatio1 = -weight1;
+ useToWeightRatio2 = -weight2;
}
return (int) Math.signum(useToWeightRatio1 - useToWeightRatio2);
@@ -225,7 +228,7 @@ public void computeSteadyShares(Collection extends FSQueue> queues,
@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
- return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare);
+ return usage.getMemorySize() > fairShare.getMemorySize();
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index fa16effd25f..a931b553152 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.util.ArrayList;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSchedulerKey;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
@@ -24,6 +25,7 @@
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -41,10 +43,13 @@
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.After;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import org.junit.Test;
public class TestSchedulerApplicationAttempt {
@@ -291,4 +296,57 @@ public void testSchedulingOpportunityOverflow() throws Exception {
assertEquals(Integer.MAX_VALUE,
app.getSchedulingOpportunities(schedulerKey));
}
+
+ @Test
+ public void testHasPendingResourceRequest() throws Exception {
+ ApplicationAttemptId attemptId = createAppAttemptId(0, 0);
+ Queue queue = createQueue("test", null);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(
+ attemptId, "user", queue, queue.getAbstractUsersManager(), rmContext);
+
+ Priority priority = Priority.newInstance(1);
+ List requests = new ArrayList<>(2);
+ Resource unit = Resource.newInstance(1L, 1);
+
+ // Add a request for a container with a node label
+ requests.add(ResourceRequest.newInstance(priority, ResourceRequest.ANY,
+ unit, 1, false, "label1"));
+ // Add a request for a container without a node label
+ requests.add(ResourceRequest.newInstance(priority, ResourceRequest.ANY,
+ unit, 1, false, ""));
+
+ // Add unique allocation IDs so that the requests aren't considered
+ // duplicates
+ requests.get(0).setAllocationRequestId(0L);
+ requests.get(1).setAllocationRequestId(1L);
+ app.updateResourceRequests(requests);
+
+ assertTrue("Reported no pending resource requests for no label when "
+ + "resource requests for no label are pending (exclusive partitions)",
+ app.hasPendingResourceRequest("",
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+ assertTrue("Reported no pending resource requests for label with pending "
+ + "resource requests (exclusive partitions)",
+ app.hasPendingResourceRequest("label1",
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+ assertFalse("Reported pending resource requests for label with no pending "
+ + "resource requests (exclusive partitions)",
+ app.hasPendingResourceRequest("label2",
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+
+ assertTrue("Reported no pending resource requests for no label when "
+ + "resource requests for no label are pending (relaxed partitions)",
+ app.hasPendingResourceRequest("",
+ SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY));
+ assertTrue("Reported no pending resource requests for label with pending "
+ + "resource requests (relaxed partitions)",
+ app.hasPendingResourceRequest("label1",
+ SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY));
+ assertTrue("Reported no pending resource requests for label with no "
+ + "pending resource requests (relaxed partitions)",
+ app.hasPendingResourceRequest("label2",
+ SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY));
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index 36ff85e5a46..5441b644181 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
@@ -21,7 +21,6 @@
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -29,12 +28,13 @@
* Dummy implementation of Schedulable for unit testing.
*/
public class FakeSchedulable implements Schedulable {
- private Resource usage;
- private Resource minShare;
- private Resource maxShare;
+ private final Resource usage;
+ private final Resource demand;
+ private final Resource minShare;
+ private final Resource maxShare;
+ private final ResourceWeights weights;
+ private final Priority priority;
private Resource fairShare;
- private ResourceWeights weights;
- private Priority priority;
private long startTime;
public FakeSchedulable() {
@@ -74,10 +74,11 @@ public FakeSchedulable(Resource minShare, Resource maxShare,
this.minShare = minShare;
this.maxShare = maxShare;
this.weights = weight;
- setFairShare(fairShare);
this.usage = usage;
+ this.demand = Resources.multiply(usage, 2.0);
this.priority = Records.newRecord(Priority.class);
- this.startTime = startTime;
+ setFairShare(fairShare);
+ start(startTime);
}
@Override
@@ -91,13 +92,13 @@ public Resource getFairShare() {
}
@Override
- public void setFairShare(Resource fairShare) {
+ public final void setFairShare(Resource fairShare) {
this.fairShare = fairShare;
}
@Override
public Resource getDemand() {
- return null;
+ return demand;
}
@Override
@@ -142,4 +143,8 @@ public void updateDemand() {}
public boolean isPreemptable() {
return true;
}
+
+ public final void start(long time) {
+ startTime = time;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index c8b9ad8bdef..9ffc5ff98e5 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -238,7 +238,7 @@ public void testAllocationFileParsing() throws Exception {
queueConf.getMaxResources("root.queueA"));
assertEquals(Resources.createResource(5120, 110),
queueConf.getMaxResources("root.queueB"));
- assertEquals(Resources.createResource(5120, 0),
+ assertEquals(Resources.createResource(5120, 100),
queueConf.getMaxResources("root.queueC"));
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueD"));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestFairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestFairSharePolicy.java
new file mode 100644
index 00000000000..becb9dc8019
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestFairSharePolicy.java
@@ -0,0 +1,321 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
+
+import java.util.Comparator;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * comparator.compare(sched1, sched2) < 0 means that sched1 should get a
+ * container before sched2
+ */
+public class TestFairSharePolicy {
+
+ private Comparator createComparator(int clusterMem,
+ int clusterCpu) {
+ FairSharePolicy policy = new FairSharePolicy();
+ FSContext fsContext = mock(FSContext.class);
+ when(fsContext.getClusterResource()).
+ thenReturn(Resources.createResource(clusterMem, clusterCpu));
+ policy.initialize(fsContext);
+ return policy.getComparator();
+ }
+
+ private FakeSchedulable createSchedulable(int memUsage, int cpuUsage) {
+ return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL, 0, 0);
+ }
+
+ private FakeSchedulable createSchedulable(int memUsage, int cpuUsage,
+ int minMemShare, int minCpuShare) {
+ return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL,
+ minMemShare, minCpuShare);
+ }
+
+ private FakeSchedulable createSchedulable(int memUsage, int cpuUsage,
+ ResourceWeights weights) {
+ return createSchedulable(memUsage, cpuUsage, weights, 0, 0);
+ }
+
+
+ private FakeSchedulable createSchedulable(int memUsage, int cpuUsage,
+ ResourceWeights weights, int minMemShare, int minCpuShare) {
+ Resource usage = BuilderUtils.newResource(memUsage, cpuUsage);
+ Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare);
+ return new FakeSchedulable(minShare,
+ Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE),
+ weights, Resources.none(), usage, 0l);
+ }
+
+ @Test
+ public void testCompareDemand() {
+ // The FakeSchedulable 2x the usage as the demand, so order is preserved
+ assertEquals("compare() should return -1 when the second request has no "
+ + "memory", -1,
+ createComparator(8000, 4).compare(createSchedulable(1000, 1),
+ createSchedulable(0, 1)));
+ assertEquals("compare() should return 1 when the first request has no "
+ + "memory", 1,
+ createComparator(8000, 4).compare(createSchedulable(0, 1),
+ createSchedulable(2000, 1)));
+ }
+
+ @Test
+ public void testCompareMinShareOneNeedy() {
+ assertEquals("compare() should return -1 when only the first request is "
+ + "below min share", -1,
+ createComparator(8000, 8).compare(createSchedulable(2000, 5, 3000, 0),
+ createSchedulable(4000, 3, 0, 0)));
+ assertEquals("compare() should return -1 when only the first request is "
+ + "below min share", -1,
+ createComparator(8000, 8).compare(createSchedulable(2000, 5, 3000, 1),
+ createSchedulable(4000, 3, 1000, 1)));
+ assertEquals("compare() should return 1 when only the second request is "
+ + "below min share", 1,
+ createComparator(8000, 8).compare(createSchedulable(2000, 5, 0, 0),
+ createSchedulable(4000, 3, 5000, 0)));
+ assertEquals("compare() should return 1 when only the second request is "
+ + "below min share", 1,
+ createComparator(8000, 8).compare(createSchedulable(2000, 5, 1000, 1),
+ createSchedulable(4000, 3, 5000, 1)));
+ }
+
+ @Test
+ public void testCompareMinShareBothNeedy() {
+ // Because demand is reported as twice the usage by FakeScheduler, we have
+ // to pick a usage and min share where usage < min share <= 2x usage
+ assertEquals("compare() should return -1 when both requests are needy, but "
+ + "the first is needier", -1,
+ createComparator(8000, 100).compare(
+ // min share is 3/5
+ createSchedulable(3000, 5, 5000, 0),
+ // min share is 4/5
+ createSchedulable(4000, 3, 5000, 0)));
+ assertEquals("compare() should return -1 when both requests are needy, but "
+ + "the first is needier", -1,
+ createComparator(8000, 100).compare(
+ // min share is 3/5
+ createSchedulable(3000, 5, 5000, 10),
+ // min share is 4/5
+ createSchedulable(4000, 3, 5000, 1)));
+ assertEquals("compare() should return 1 when both requests are needy, but "
+ + "the second is needier", 1,
+ createComparator(8000, 100).compare(
+ // min share is 4/5
+ createSchedulable(4000, 5, 5000, 0),
+ // min share is 3/5
+ createSchedulable(3000, 3, 5000, 0)));
+ assertEquals("compare() should return 1 when both requests are needy, but "
+ + "the second is needier", 1,
+ createComparator(8000, 100).compare(
+ // min share is 4/5
+ createSchedulable(4000, 5, 5000, 1),
+ // min share is 3/5
+ createSchedulable(3000, 3, 5000, 10)));
+ }
+
+ @Test
+ public void testCompareDemandWithEqualWeights() {
+ FakeSchedulable s1;
+ FakeSchedulable s2;
+
+ s1 = createSchedulable(1000, 0);
+ s2 = createSchedulable(2000, 0);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weights are equal, and the first is further below target share",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 10);
+ s2 = createSchedulable(2000, 1);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weights are equal, and the first is further below target share",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 0);
+ s2 = createSchedulable(1000, 0);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, and the second is further below target share",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 10);
+ s2 = createSchedulable(1000, 1);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, and the second is further below target share",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 0);
+ s2 = createSchedulable(1000, 0);
+ s1.start(10L);
+ s2.start(1L);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, target shares are equal, and the second one "
+ + "was started first",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 1);
+ s2 = createSchedulable(1000, 10);
+ s1.start(10L);
+ s2.start(1L);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, target shares are equal, and the second one "
+ + "was started first",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 0);
+ s2 = createSchedulable(1000, 0);
+ s1.start(1L);
+ s2.start(10L);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weights are equal, target shares are equal, and the first one "
+ + "was started first",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 10);
+ s2 = createSchedulable(1000, 1);
+ s1.start(1L);
+ s2.start(10L);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weights are equal, target shares are equal, and the first one "
+ + "was started first",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 0);
+ s2 = createSchedulable(1000, 0);
+ assertEquals("compare() should return the same result as comparing the "
+ + "schedulables' names when neither request is needy, "
+ + "the weights are equal, and the target shares are equal",
+ s1.getName().compareTo(s2.getName()),
+ createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 10);
+ s2 = createSchedulable(1000, 1);
+ assertEquals("compare() should return the same result as comparing the "
+ + "schedulables' names when neither request is needy, "
+ + "the weights are equal, and the target shares are equal",
+ s1.getName().compareTo(s2.getName()),
+ createComparator(8000, 8).compare(s1, s2));
+ }
+
+ @Test
+ public void testCompareDemandWithUnequalWeights() {
+ FakeSchedulable s1;
+ FakeSchedulable s2;
+
+ s1 = createSchedulable(1000, 0, new ResourceWeights(3.0f));
+ s2 = createSchedulable(2000, 0);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "and the first is further below weighted target share",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 10, new ResourceWeights(3.0f));
+ s2 = createSchedulable(2000, 1);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "and the first is further below weighted target share",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 10);
+ s2 = createSchedulable(1000, 1, new ResourceWeights(3.0f));
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "and the second is further below weighted target share",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 10);
+ s2 = createSchedulable(1000, 1, new ResourceWeights(3.0f));
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "and the second is further below weighted target share",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 0, new ResourceWeights(3.0f));
+ s2 = createSchedulable(6000, 1);
+ s1.start(1L);
+ s2.start(10L);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weighted target shares are equal, and the first one was "
+ + "started first",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(1000, 10, new ResourceWeights(3.0f));
+ s2 = createSchedulable(6000, 1);
+ s1.start(1L);
+ s2.start(10L);
+ assertEquals("compare() should return -1 when neither request is needy, "
+ + "the weighted target shares are equal, and the first one was "
+ + "started first",
+ -1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 0, new ResourceWeights(2.0f));
+ s2 = createSchedulable(1000, 1);
+ s1.start(10L);
+ s2.start(1L);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weighted target shares are equal, and the second one was "
+ + "started first",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 10, new ResourceWeights(2.0f));
+ s2 = createSchedulable(1000, 1);
+ s1.start(10L);
+ s2.start(1L);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weighted target shares are equal, and the second one was "
+ + "started first",
+ 1, createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 0, new ResourceWeights(2.0f));
+ s2 = createSchedulable(1000, 1);
+ assertEquals("compare() should return the same result as comparing the "
+ + "schedulables' names when neither request is needy, "
+ + "and the weighted target shares are equal",
+ s1.getName().compareTo(s2.getName()),
+ createComparator(8000, 8).compare(s1, s2));
+
+ s1 = createSchedulable(2000, 10, new ResourceWeights(2.0f));
+ s2 = createSchedulable(1000, 1);
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weighted target shares are equal, and the second one was "
+ + "started first",
+ s1.getName().compareTo(s2.getName()),
+ createComparator(8000, 8).compare(s1, s2));
+ }
+
+ @Test
+ public void testCompareSchedulablesWithClusterResourceChanges(){
+ Schedulable schedulable1 = createSchedulable(2000, 1);
+ Schedulable schedulable2 = createSchedulable(1000, 2);
+
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, and the second is further below target share",
+ 1, createComparator(4000, 5).compare(schedulable1, schedulable2));
+
+ // Now try again with a different cluster size
+ assertEquals("compare() should return 1 when neither request is needy, "
+ + "the weights are equal, and the second is further below target share",
+ 1, createComparator(4000, 5).compare(schedulable1, schedulable2));
+ }
+}