diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index c98aadc..17fe100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -52,12 +52,15 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { // Max concurrent running applications for each queue and for each user; in addition, // for users that have no max specified, we use the userMaxJobsDefault. + // Max application life time for each queue. @VisibleForTesting final Map queueMaxApps; @VisibleForTesting final Map userMaxApps; + final Map queueMaxAppLifetime; private final int userMaxAppsDefault; private final int queueMaxAppsDefault; + private final long queueMaxAppLifetimeDefault; private final ConfigurableResource queueMaxResourcesDefault; // Maximum resource share for each leaf queue that can be used to run AMs @@ -124,6 +127,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { this.queueMaxAppsDefault = allocationFileParser.getQueueMaxAppsDefault(); this.queueMaxAMShareDefault = allocationFileParser.getQueueMaxAMShareDefault(); + this.queueMaxAppLifetimeDefault = allocationFileParser.getQueueMaxAppLifetimeDefault(); this.defaultSchedulingPolicy = allocationFileParser.getDefaultSchedulingPolicy(); this.schedulingPolicies = queueProperties.getQueuePolicies(); @@ -135,6 +139,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { queueProperties.getFairSharePreemptionThresholds(); this.queueAcls = queueProperties.getQueueAcls(); this.resAcls = queueProperties.getReservationAcls(); + this.queueMaxAppLifetime = queueProperties.getQueueMaxAppLifetime(); this.reservableQueues = queueProperties.getReservableQueues(); this.globalReservationQueueConfig = globalReservationQueueConfig; this.placementPolicy = newPlacementPolicy; @@ -154,8 +159,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { queueMaxAppsDefault = Integer.MAX_VALUE; queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded()); queueMaxAMShareDefault = 0.5f; + queueMaxAppLifetimeDefault = -1; queueAcls = new HashMap<>(); resAcls = new HashMap<>(); + queueMaxAppLifetime = new HashMap<>(); minSharePreemptionTimeouts = new HashMap<>(); fairSharePreemptionTimeouts = new HashMap<>(); fairSharePreemptionThresholds = new HashMap<>(); @@ -263,6 +270,12 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare; } + long getQueueMaxAppLifetime(String queue) { + Long maxAppLifetime = queueMaxAppLifetime.get(queue); + return (maxAppLifetime == null) ? queueMaxAppLifetimeDefault : maxAppLifetime; + + } + /** * Get the minimum resource allocation for the given queue. * @@ -395,6 +408,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { queue.setMaxRunningApps(getQueueMaxApps(name)); queue.setMaxAMShare(getQueueMaxAMShare(name)); queue.setMaxChildQueueResource(getMaxChildResources(name)); + queue.setMaxAppLifetime(getQueueMaxAppLifetime(name)); // Set queue metrics. queue.getMetrics().setMinShare(queue.getMinShare()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 4babfd5..1fcaedf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -74,6 +74,7 @@ public abstract class FSQueue implements Queue, Schedulable { protected Resource minShare; private ConfigurableResource maxShare; protected int maxRunningApps; + protected long maxAppLifetime; private ConfigurableResource maxChildQueueResource; // maxAMShare is a value between 0 and 1. @@ -198,6 +199,10 @@ public abstract class FSQueue implements Queue, Schedulable { return maxRunningApps; } + public void setMaxAppLifetime(long maxAppLifetime) { this.maxAppLifetime = maxAppLifetime; } + + public long getMaxAppLifetime() { return maxAppLifetime; } + @VisibleForTesting protected float getMaxAMShare() { return maxAMShare; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1f85814..9b4d162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1859,6 +1860,31 @@ public class FairScheduler extends @Override public long checkAndGetApplicationLifetime(String queueName, long lifetime) { // Lifetime is the application lifetime by default. - return lifetime; + try { + readLock.lock(); + FSQueue queue = queueMgr.getQueue(queueName); + + if (queue == null || !(queue instanceof FSLeafQueue)) { + return lifetime; + } + + long maxAppLifetime = + queue.getMaxAppLifetime(); + + // this is the default max App lifetime + // or the max App lifetime we set + if (maxAppLifetime <= 0) { + return lifetime; + } + + if (lifetime <= 0 || lifetime > maxAppLifetime) { + return maxAppLifetime; + } + return lifetime; + } finally { + readLock.unlock(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileParser.java index 161405b..be8fd2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileParser.java @@ -62,6 +62,8 @@ public class AllocationFileParser { "defaultFairSharePreemptionThreshold"; private static final String QUEUE_MAX_AM_SHARE_DEFAULT = "queueMaxAMShareDefault"; + private static final String QUEUE_MAX_APP_LIFETIME_DEFAULT = + "queueMaxAppLifetimeDefault"; private static final String RESERVATION_PLANNER = "reservation-planner"; private static final String RESERVATION_AGENT = "reservation-agent"; private static final String RESERVATION_ADMISSION_POLICY = @@ -83,7 +85,7 @@ public class AllocationFileParser { DEFAULT_MIN_SHARE_PREEMPTION_TIMEOUT, QUEUE_MAX_APPS_DEFAULT, DEFAULT_FAIR_SHARE_PREEMPTION_THRESHOLD, QUEUE_MAX_AM_SHARE_DEFAULT, RESERVATION_PLANNER, RESERVATION_AGENT, RESERVATION_ADMISSION_POLICY, - QUEUE_PLACEMENT_POLICY, QUEUE, POOL, USER, + QUEUE_PLACEMENT_POLICY, QUEUE_MAX_APP_LIFETIME_DEFAULT, QUEUE, POOL, USER, DEFAULT_QUEUE_SCHEDULING_POLICY, DEFAULT_QUEUE_SCHEDULING_MODE); private final NodeList elements; @@ -220,6 +222,11 @@ public class AllocationFileParser { return 0.5f; } + public long getQueueMaxAppLifetimeDefault() { + Optional value = getTextValue(QUEUE_MAX_APP_LIFETIME_DEFAULT); + return value.map(Long :: parseLong ).orElse(-1L); + } + // Reservation global configuration knobs public Optional getReservationPlanner() { return getTextValue(RESERVATION_PLANNER); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java index ec7e4a4..7248b03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java @@ -50,6 +50,7 @@ public class AllocationFileQueueParser { private static final String MAX_RESOURCES = "maxResources"; private static final String MAX_CHILD_RESOURCES = "maxChildResources"; private static final String MAX_RUNNING_APPS = "maxRunningApps"; + private static final String MAX_APP_LIFETIME = "maxAppLifetime"; private static final String MAX_AMSHARE = "maxAMShare"; private static final String WEIGHT = "weight"; private static final String MIN_SHARE_PREEMPTION_TIMEOUT = @@ -150,7 +151,11 @@ public class AllocationFileQueueParser { String text = getTrimmedTextData(field); int val = Integer.parseInt(text); builder.queueMaxApps(queueName, val); - } else if (MAX_AMSHARE.equals(field.getTagName())) { + } else if (MAX_APP_LIFETIME.equals(field.getTagName())) { + String text = getTrimmedTextData(field); + int val = Integer.parseInt(text); + builder.queueMaxAppLifetime(queueName, val); + }else if (MAX_AMSHARE.equals(field.getTagName())) { String text = getTrimmedTextData(field); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java index ee5f179..add9abb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java @@ -44,6 +44,7 @@ public class QueueProperties { private final Map queueMaxAMShares; private final Map queueWeights; private final Map queuePolicies; + private final Map queueMaxAppLifetime; private final Map minSharePreemptionTimeouts; private final Map fairSharePreemptionTimeouts; private final Map fairSharePreemptionThresholds; @@ -63,6 +64,7 @@ public class QueueProperties { this.configuredQueues = builder.configuredQueues; this.queueMaxAMShares = builder.queueMaxAMShares; this.queuePolicies = builder.queuePolicies; + this.queueMaxAppLifetime = builder.queueMaxAppLifetime; this.fairSharePreemptionThresholds = builder.fairSharePreemptionThresholds; this.queueMaxApps = builder.queueMaxApps; this.minSharePreemptionTimeouts = builder.minSharePreemptionTimeouts; @@ -116,6 +118,8 @@ public class QueueProperties { return queuePolicies; } + public Map getQueueMaxAppLifetime() { return queueMaxAppLifetime; } + public Map> getQueueAcls() { return queueAcls; } @@ -151,6 +155,7 @@ public class QueueProperties { private Map queueMaxAMShares = new HashMap<>(); private Map queueWeights = new HashMap<>(); private Map queuePolicies = new HashMap<>(); + private Map queueMaxAppLifetime = new HashMap<>(); private Map minSharePreemptionTimeouts = new HashMap<>(); private Map fairSharePreemptionTimeouts = new HashMap<>(); private Map fairSharePreemptionThresholds = new HashMap<>(); @@ -213,6 +218,11 @@ public class QueueProperties { return this; } + public Builder queueMaxAppLifetime(String queueName, long value) { + this.queueMaxAppLifetime.put(queueName, value); + return this; + } + public Builder minSharePreemptionTimeouts(String queueName, long value) { this.minSharePreemptionTimeouts.put(queueName, value); return this;