diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index fe78de2..036b6e8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @Private @Unstable @@ -64,6 +65,10 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { + if (stepFactor.getMemorySize() == 0) { + throw new YarnRuntimeException("StepFactor memory size cannot be zero!"); + } + long normalizedMemory = Math.min( roundUp( Math.max(r.getMemorySize(), minimumResource.getMemorySize()), @@ -73,12 +78,6 @@ public Resource normalize(Resource r, Resource minimumResource, } @Override - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - return normalize(r, minimumResource, maximumResource, minimumResource); - } - - @Override public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( roundUp(r.getMemorySize(), stepFactor.getMemorySize()) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 2695811..be3100a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; /** * A {@link ResourceCalculator} which uses the concept of @@ -152,6 +153,12 @@ public Resource divideAndCeil(Resource numerator, int denominator) { @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { + // Check memory and vcore independently since we should make sure each of + // them cannot be zero. + if (stepFactor.getMemorySize() == 0 || stepFactor.getVirtualCores() == 0) { + throw new YarnRuntimeException("StepFactor resource cannot be zero!"); + } + long normalizedMemory = Math.min( roundUp( Math.max(r.getMemorySize(), minimumResource.getMemorySize()), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index a7df9c5..a1800e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -99,21 +99,6 @@ public abstract Resource multiplyAndNormalizeDown( /** * Normalize resource r given the base * minimumResource and verify against max allowed - * maximumResource - * - * @param r resource - * @param minimumResource step-factor - * @param maximumResource the upper bound of the resource to be allocated - * @return normalized resource - */ - public Resource normalize(Resource r, Resource minimumResource, - Resource maximumResource) { - return normalize(r, minimumResource, maximumResource, minimumResource); - } - - /** - * Normalize resource r given the base - * minimumResource and verify against max allowed * maximumResource using a step factor for hte normalization. * * @param r resource diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index c065b60..c6d2966 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -455,11 +455,7 @@ private ResourceRequest validateAndCreateResourceRequest( throw e; } - SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), - scheduler.getClusterResource(), - scheduler.getMinimumResourceCapability(), - scheduler.getMaximumResourceCapability(), - scheduler.getMinimumResourceCapability()); + scheduler.normalizeRequest(amReq); return amReq; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index df59556..1f5d8d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -1004,4 +1004,20 @@ protected synchronized void nodeUpdate(RMNode nm) { " availableResource: " + node.getUnallocatedResource()); } } + + @Override + public void normalizeRequest(ResourceRequest ask) { + SchedulerUtils.normalizeRequest(ask, + getResourceCalculator(), + getClusterResource(), + getMinimumResourceCapability(), + getMaximumResourceCapability(), + getMinimumResourceCapability()); + } + + protected void normalizeRequest(List asks) { + for (ResourceRequest ask: asks) { + normalizeRequest(ask); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c999e26..0a34c2a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; -import java.util.List; import java.util.Set; import org.apache.commons.lang.StringUtils; @@ -36,6 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessType; @@ -124,23 +124,6 @@ private static ContainerStatus createAbnormalContainerStatus( } /** - * Utility method to normalize a list of resource requests, by insuring that - * the memory for each request is a multiple of minMemory and is not zero. - */ - public static void normalizeRequests( - List asks, - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource minimumResource, - Resource maximumResource) { - for (ResourceRequest ask : asks) { - normalizeRequest( - ask, resourceCalculator, clusterResource, minimumResource, - maximumResource, minimumResource); - } - } - - /** * Utility method to normalize a resource request, by insuring that the * requested memory is a multiple of minMemory and is not zero. */ @@ -150,43 +133,32 @@ public static void normalizeRequest( Resource clusterResource, Resource minimumResource, Resource maximumResource) { - Resource normalized = - Resources.normalize( - resourceCalculator, ask.getCapability(), minimumResource, - maximumResource, minimumResource); - ask.setCapability(normalized); + normalizeRequest(ask, resourceCalculator, clusterResource, + minimumResource, maximumResource, minimumResource); } /** - * Utility method to normalize a list of resource requests, by insuring that - * the memory for each request is a multiple of minMemory and is not zero. - */ - public static void normalizeRequests( - List asks, - ResourceCalculator resourceCalculator, - Resource clusterResource, - Resource minimumResource, - Resource maximumResource, - Resource incrementResource) { - for (ResourceRequest ask : asks) { - normalizeRequest( - ask, resourceCalculator, clusterResource, minimumResource, - maximumResource, incrementResource); - } - } - - /** * Utility method to normalize a resource request, by insuring that the - * requested memory is a multiple of minMemory and is not zero. + * requested memory is a multiple of increment resource and is not zero. */ public static void normalizeRequest( ResourceRequest ask, - ResourceCalculator resourceCalculator, + ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) { - Resource normalized = + if (Resources.equals(incrementResource, Resources.none())) { + throw new YarnRuntimeException("Increment resource cannot be zero!"); + } + + if (Resources.greaterThan(resourceCalculator, clusterResource, + minimumResource, maximumResource)) { + throw new YarnRuntimeException("Minimum resource cannot be larger than " + + "maximum Resource"); + } + + Resource normalized = Resources.normalize( resourceCalculator, ask.getCapability(), minimumResource, maximumResource, incrementResource); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index c4f575f..3405c73 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -361,4 +361,11 @@ public void updateApplicationPriority(Priority newPriority, * @return SchedulerNode corresponds to nodeId */ SchedulerNode getSchedulerNode(NodeId nodeId); + + /** + * Normalize a resource request. + * + * @param request the resource request to be normalized + */ + void normalizeRequest(ResourceRequest request); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cfdcb10..c417f3d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1015,9 +1015,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, decreaseContainers(decreaseRequests, application); // Sanity check for new allocation requests - SchedulerUtils.normalizeRequests(ask, getResourceCalculator(), - getClusterResource(), getMinimumResourceCapability(), - getMaximumResourceCapability()); + normalizeRequest(ask); Allocation allocation; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 94fdb7c..e763029 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -963,6 +963,16 @@ private void removeNode(RMNode rmNode) { } @Override + public void normalizeRequest(ResourceRequest ask) { + SchedulerUtils.normalizeRequest(ask, + DOMINANT_RESOURCE_CALCULATOR, + getClusterResource(), + minimumAllocation, + getMaximumResourceCapability(), + incrAllocation); + } + + @Override public Allocation allocate(ApplicationAttemptId appAttemptId, List ask, List release, List blacklistAdditions, List blacklistRemovals, @@ -978,9 +988,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, } // Sanity check - SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR, - getClusterResource(), minimumAllocation, getMaximumResourceCapability(), - incrAllocation); + normalizeRequest(ask); // Record container allocation start time application.recordContainerRequestTime(getClock().getTime()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 92acf75..4f5e321 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -336,9 +336,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, } // Sanity check - SchedulerUtils.normalizeRequests(ask, resourceCalculator, - getClusterResource(), minimumAllocation, - getMaximumResourceCapability()); + normalizeRequest(ask); // Release containers releaseContainers(release, application);