diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 0ea73140..f005d017 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -54,6 +54,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { private final int userMaxAppsDefault; private final int queueMaxAppsDefault; + private final Resource appMaxReserveResources; + // Maximum resource share for each leaf queue that can be used to run AMs final Map queueMaxAMShares; private final float queueMaxAMShareDefault; @@ -109,7 +111,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { QueuePlacementPolicy placementPolicy, Map> configuredQueues, ReservationQueueConfiguration globalReservationQueueConfig, - Set reservableQueues) { + Set reservableQueues, Resource appMaxReserveResources) { this.minQueueResources = minQueueResources; this.maxQueueResources = maxQueueResources; this.queueMaxApps = queueMaxApps; @@ -129,6 +131,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { this.globalReservationQueueConfig = globalReservationQueueConfig; this.placementPolicy = placementPolicy; this.configuredQueues = configuredQueues; + this.appMaxReserveResources = appMaxReserveResources; } public AllocationConfiguration(Configuration conf) { @@ -154,6 +157,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { } placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); + appMaxReserveResources = Resources.unbounded(); } /** @@ -227,6 +231,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare; } + public Resource getAppMaxReserveResources() { + return (appMaxReserveResources == null) ? Resources.unbounded() : appMaxReserveResources; + } + /** * Get the minimum resource allocation for the given queue. * @return the cap set on this queue, or 0 if not set. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index ce393a49..896507e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -323,6 +323,8 @@ public class AllocationFileLoaderService extends AbstractService { long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; float defaultFairSharePreemptionThreshold = 0.5f; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; + Resource appMaxReserveResources = Resources.unbounded(); + // Reservation global configuration knobs String planner = null; @@ -414,6 +416,10 @@ public class AllocationFileLoaderService extends AbstractService { || "defaultQueueSchedulingMode".equals(element.getTagName())) { String text = ((Text)element.getFirstChild()).getData().trim(); defaultSchedPolicy = SchedulingPolicy.parse(text); + } else if("appMaxReserveResources".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); + appMaxReserveResources = val; } else if ("queuePlacementPolicy".equals(element.getTagName())) { placementPolicyElement = element; } else if ("reservation-planner".equals(element.getTagName())) { @@ -495,7 +501,7 @@ public class AllocationFileLoaderService extends AbstractService { minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, - reservableQueues); + reservableQueues, appMaxReserveResources); lastSuccessfulReload = clock.getTime(); lastReloadAttemptFailed = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 87121850..b5ae9683 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -556,7 +556,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) { return Resources.none(); } - + Resource appMaxReserveResources = scheduler.getAllocationConfiguration().getAppMaxReserveResources(); + Resource currentReservation = getCurrentReservation(); + Resource reservationPlusCapability = Resources.add(currentReservation, capability); + if(!Resources.fitsIn(reservationPlusCapability, appMaxReserveResources)) { + return Resources.none(); + } // The desired container won't fit here, so reserve reserve(request.getPriority(), node, container, reserved);