diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index c065b60..c6d2966 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -455,11 +455,7 @@ private ResourceRequest validateAndCreateResourceRequest( throw e; } - SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), - scheduler.getClusterResource(), - scheduler.getMinimumResourceCapability(), - scheduler.getMaximumResourceCapability(), - scheduler.getMinimumResourceCapability()); + scheduler.normalizeRequest(amReq); return amReq; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index df59556..10c1c08 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -1004,4 +1004,11 @@ protected synchronized void nodeUpdate(RMNode nm) { " availableResource: " + node.getUnallocatedResource()); } } + + @Override + public void normalizeRequest(ResourceRequest ask) { + SchedulerUtils.normalizeRequest(ask, getResourceCalculator(), + getMinimumResourceCapability(), getMaximumResourceCapability(), + getMinimumResourceCapability()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c999e26..1b100b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -124,69 +124,33 @@ private static ContainerStatus createAbnormalContainerStatus( } /** - * Utility method to normalize a list of resource requests, by insuring that - * the memory for each request is a multiple of minMemory and is not zero. - */ - public static void normalizeRequests( - List asks, - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource minimumResource, - Resource maximumResource) { - for (ResourceRequest ask : asks) { - normalizeRequest( - ask, resourceCalculator, clusterResource, minimumResource, - maximumResource, minimumResource); - } - } - - /** * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of minMemory and is not zero. */ public static void normalizeRequest( ResourceRequest ask, ResourceCalculator resourceCalculator, - Resource clusterResource, Resource minimumResource, Resource maximumResource) { - Resource normalized = - Resources.normalize( - resourceCalculator, ask.getCapability(), minimumResource, - maximumResource, minimumResource); - ask.setCapability(normalized); + normalizeRequest(ask, resourceCalculator, minimumResource, maximumResource, + minimumResource); } /** - * Utility method to normalize a list of resource requests, by insuring that - * the memory for each request is a multiple of minMemory and is not zero. - */ - public static void normalizeRequests( - List asks, - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource minimumResource, - Resource maximumResource, - Resource incrementResource) { - for (ResourceRequest ask : asks) { - normalizeRequest( - ask, resourceCalculator, clusterResource, minimumResource, - maximumResource, incrementResource); - } - } - - /** * Utility method to normalize a resource request, by insuring that the - * requested memory is a multiple of minMemory and is not zero. + * requested memory is a multiple of increment resource and is not zero. */ public static void normalizeRequest( ResourceRequest ask, ResourceCalculator resourceCalculator, - Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) { - Resource normalized = + if (Resources.equals(incrementResource, Resources.none())) { + throw new RuntimeException("Increment resource cannot be zero!"); + } + + Resource normalized = Resources.normalize( resourceCalculator, ask.getCapability(), minimumResource, maximumResource, incrementResource); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index c4f575f..3405c73 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -361,4 +361,11 @@ public void updateApplicationPriority(Priority newPriority, * @return SchedulerNode corresponds to nodeId */ SchedulerNode getSchedulerNode(NodeId nodeId); + + /** + * Normalize a resource request. + * + * @param request the resource request to be normalized + */ + void normalizeRequest(ResourceRequest request); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cfdcb10..a9f79d5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1015,9 +1015,9 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, decreaseContainers(decreaseRequests, application); // Sanity check for new allocation requests - SchedulerUtils.normalizeRequests(ask, getResourceCalculator(), - getClusterResource(), getMinimumResourceCapability(), - getMaximumResourceCapability()); + for (ResourceRequest item : ask) { + normalizeRequest(item); + } Allocation allocation; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 94fdb7c..2f8f072 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -963,6 +963,12 @@ private void removeNode(RMNode rmNode) { } @Override + public void normalizeRequest(ResourceRequest ask) { + SchedulerUtils.normalizeRequest(ask, DOMINANT_RESOURCE_CALCULATOR, + minimumAllocation, getMaximumResourceCapability(), incrAllocation); + } + + @Override public Allocation allocate(ApplicationAttemptId appAttemptId, List ask, List release, List blacklistAdditions, List blacklistRemovals, @@ -978,9 +984,9 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, } // Sanity check - SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR, - getClusterResource(), minimumAllocation, getMaximumResourceCapability(), - incrAllocation); + for (ResourceRequest item : ask) { + normalizeRequest(item); + } // Record container allocation start time application.recordContainerRequestTime(getClock().getTime()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 92acf75..349a90b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -336,9 +336,9 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, } // Sanity check - SchedulerUtils.normalizeRequests(ask, resourceCalculator, - getClusterResource(), minimumAllocation, - getMaximumResourceCapability()); + for (ResourceRequest item : ask) { + normalizeRequest(item); + } // Release containers releaseContainers(release, application); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 63f97c5..216d66c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -112,37 +112,37 @@ public void testNormalizeRequest() { // case negative memory ask.setCapability(Resources.createResource(-1024)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(minMemory, ask.getCapability().getMemorySize()); // case zero memory ask.setCapability(Resources.createResource(0)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(minMemory, ask.getCapability().getMemorySize()); // case memory is a multiple of minMemory ask.setCapability(Resources.createResource(2 * minMemory)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(2 * minMemory, ask.getCapability().getMemorySize()); // case memory is not a multiple of minMemory ask.setCapability(Resources.createResource(minMemory + 10)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(2 * minMemory, ask.getCapability().getMemorySize()); // case memory is equal to max allowed ask.setCapability(Resources.createResource(maxMemory)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(maxMemory, ask.getCapability().getMemorySize()); // case memory is just less than max ask.setCapability(Resources.createResource(maxMemory - 10)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(maxMemory, ask.getCapability().getMemorySize()); @@ -150,18 +150,18 @@ public void testNormalizeRequest() { maxResource = Resources.createResource(maxMemory - 10, 0); ask.setCapability(Resources.createResource(maxMemory - 100)); // multiple of minMemory > maxMemory, then reduce to maxMemory - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize()); // ask is more than max maxResource = Resources.createResource(maxMemory, 0); ask.setCapability(Resources.createResource(maxMemory + 100)); - SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, + SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource); assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize()); } - + @Test (timeout = 30000) public void testNormalizeRequestWithDominantResourceCalculator() { ResourceCalculator resourceCalculator = new DominantResourceCalculator(); @@ -175,13 +175,13 @@ public void testNormalizeRequestWithDominantResourceCalculator() { // case negative memory/vcores ask.setCapability(Resources.createResource(-1024, -1)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, minResource, maxResource); assertEquals(minResource, ask.getCapability()); // case zero memory/vcores ask.setCapability(Resources.createResource(0, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, minResource, maxResource); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemorySize()); @@ -189,7 +189,7 @@ public void testNormalizeRequestWithDominantResourceCalculator() { // case non-zero memory & zero cores ask.setCapability(Resources.createResource(1536, 0)); SchedulerUtils.normalizeRequest( - ask, resourceCalculator, clusterResource, minResource, maxResource); + ask, resourceCalculator, minResource, maxResource); assertEquals(Resources.createResource(2048, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(2048, ask.getCapability().getMemorySize());