diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b23714b..34ef583 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -19,6 +19,9 @@ Release 2.7.1 - UNRELEASED YARN-3469. ZKRMStateStore: Avoid setting watches that are not required. (Jun Gong via kasha) + YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue + info once. (Varun Saxena via wangda) + BUG FIXES YARN-3487. CapacityScheduler scheduler lock obtained unnecessarily when diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 083189a..90c17c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -93,9 +94,16 @@ public static void normalizeAndValidateRequests(List ask, Resource maximumResource, String queueName, YarnScheduler scheduler) throws InvalidResourceRequestException { + // Get queue from scheduler + QueueInfo queueInfo = null; + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + } + for (ResourceRequest resReq : ask) { SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, - queueName, scheduler); + queueName, scheduler, queueInfo); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 432ac82..17f4bfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -213,13 +213,21 @@ public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery) throws InvalidResourceRequestException { + normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, + isRecovery, null); + } - QueueInfo queueInfo = null; - try { - queueInfo = scheduler.getQueueInfo(queueName, false, false); - } catch (IOException e) { - // it is possible queue cannot get when queue mapping is set, just ignore - // the queueInfo here, and move forward + public static void normalizeAndValidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + boolean isRecovery, QueueInfo queueInfo) + throws InvalidResourceRequestException { + if (null == queueInfo) { + try { + queueInfo = scheduler.getQueueInfo(queueName, false, false); + } catch (IOException e) { + // it is possible queue cannot get when queue mapping is set, just ignore + // the queueInfo here, and move forward + } } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { @@ -230,8 +238,16 @@ public static void normalizeAndValidateRequest(ResourceRequest resReq, public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler) throws InvalidResourceRequestException { + normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler, + null); + } + + public static void normalizeAndvalidateRequest(ResourceRequest resReq, + Resource maximumResource, String queueName, YarnScheduler scheduler, + QueueInfo queueInfo) + throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, - false); + false, queueInfo); } /**