Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java (revision 55fad6a3de3125d9e7e2e9a5f8fa5b1b22a1de60) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java (revision c038b78435ef77841bed573577fe2fe630d4e3d8) @@ -111,6 +111,10 @@ private final Map allowedLocalityLevel = new HashMap<>(); + private float weight = 0; + private long weightMemory = -1; + private int weightPriority = -1; + public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { @@ -1325,14 +1329,27 @@ @Override public float getWeight() { - float weight = 1.0F; - - if (scheduler.isSizeBasedWeight()) { - // Set weight based on current memory demand - weight = (float)(Math.log1p(demand.getMemorySize()) / Math.log(2)); - } + // Spare some cpu time not recalculating the + // difficult stuff + boolean sizeBased = scheduler.isSizeBasedWeight(); + while(true) { + long tmpMemory = demand.getMemorySize(); + int tmpPriority = appPriority.getPriority(); + if (weightMemory == tmpMemory && + weightPriority == tmpPriority) { + break; + } + float tmpWeight = tmpPriority; + if (sizeBased) { + // Set weight based on current memory demand + tmpWeight *= (float) (Math.log1p(tmpMemory) / Math.log(2)); + } - return weight * appPriority.getPriority(); + weight = tmpWeight; + weightMemory = tmpMemory; + weightPriority = tmpPriority; + } + return weight; } @Override Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java (revision 55fad6a3de3125d9e7e2e9a5f8fa5b1b22a1de60) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java (revision c038b78435ef77841bed573577fe2fe630d4e3d8) @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.SIZE_BASED_WEIGHT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -58,6 +59,7 @@ @Before public void setup() { Configuration conf = createConfiguration(); + conf.setBoolean(SIZE_BASED_WEIGHT, true); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); @@ -351,4 +353,30 @@ assertEquals(expectedMemory, headroom.getMemorySize()); assertEquals(expectedCPU, headroom.getVirtualCores()); } + + @Test + public void testWeight() { + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + RMContext rmContext = resourceManager.getRMContext(); + FSAppAttempt schedulerApp = + new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, + null, rmContext); + + Resource requestedResource = Resource.newInstance(1024, 2); + schedulerApp.getAppAttemptResourceUsage().setUsed(requestedResource); + schedulerApp.updateDemand(); + assertEquals(10, schedulerApp.getWeight(), 0.1); + assertEquals(10, schedulerApp.getWeight(), 0.1); + + Resource requestedResource2 = Resource.newInstance(2048, 2); + schedulerApp.getAppAttemptResourceUsage().setUsed(requestedResource2); + schedulerApp.updateDemand(); + assertEquals(11, schedulerApp.getWeight(), 0.1); + assertEquals(11, schedulerApp.getWeight(), 0.1); + + schedulerApp.getPriority().setPriority(2); + assertEquals(22, schedulerApp.getWeight(), 0.1); + assertEquals(22, schedulerApp.getWeight(), 0.1); + } }