From d8261666525a7c941a32a30566f8e3b2f774db12 Mon Sep 17 00:00:00 2001 From: prabhujoseph Date: Tue, 29 Oct 2019 12:24:37 +0530 Subject: [PATCH] YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. --- .../webapp/dao/CapacitySchedulerInfo.java | 11 +++-- .../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 23 +++++++-- .../webapp/dao/CapacitySchedulerQueueInfo.java | 54 +++++++++++++++++++++- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index ca42041..b8a5426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,7 +61,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); - queues = getQueues(parent); + queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); } @@ -89,7 +89,8 @@ public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues( + CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); // JAXB marashalling leads to situation where the "type" field injected @@ -112,10 +113,10 @@ protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + info = new CapacitySchedulerQueueInfo(cs, queue); + info.queues = getQueues(cs, queue); } queuesInfo.addToQueueInfoList(info); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a53e921..fe75df7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -22,13 +22,13 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -45,6 +45,7 @@ protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected float configuredMaxAMResource; protected ResourceInfo AMResourceLimit; protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; @@ -53,8 +54,8 @@ protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; - - @XmlTransient + protected long maxApplicationLifetime; + protected long defaultApplicationLifetime; protected String orderingPolicyInfo; CapacitySchedulerLeafQueueInfo() { @@ -70,6 +71,7 @@ userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsersManager().getUsersInfo()); userLimitFactor = q.getUserLimitFactor(); + configuredMaxAMResource = q.getMaxAMResourcePerQueuePercent(); AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); @@ -91,6 +93,8 @@ if ( q instanceof AutoCreatedLeafQueue) { isAutoCreatedLeafQueue = true; } + defaultApplicationLifetime = q.getDefaultApplicationLifetime(); + maxApplicationLifetime = q.getMaximumApplicationLifetime(); } @Override @@ -136,6 +140,10 @@ public UsersInfo getUsers() { public float getUserLimitFactor() { return userLimitFactor; } + + public float getConfiguredMaxAMResource() { + return configuredMaxAMResource; + } public ResourceInfo getAMResourceLimit() { return AMResourceLimit; @@ -172,4 +180,13 @@ public int getDefaultApplicationPriority() { public boolean isAutoCreatedLeafQueue() { return isAutoCreatedLeafQueue; } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index fd256b1..f1ec28a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,6 +19,8 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -27,10 +29,16 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -65,11 +73,17 @@ protected ResourcesInfo resources; protected ResourceInfo minEffectiveCapacity; protected ResourceInfo maxEffectiveCapacity; + protected ResourceInfo maximumAllocation; + protected HashMap acls = + new HashMap(); + protected String priorityAcls; + protected int queuePriority; + protected String orderingPolicyInfo; CapacitySchedulerQueueInfo() { }; - CapacitySchedulerQueueInfo(CSQueue q) { + CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; @@ -114,6 +128,24 @@ q.getQueueResourceQuotas().getEffectiveMinResource()); maxEffectiveCapacity = new ResourceInfo( q.getQueueResourceQuotas().getEffectiveMaxResource()); + maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + for (Map.Entry e : conf + .getAcls(queueName).entrySet()) { + acls.put(e.getKey().toString(), e.getValue().getAclString()); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + priorityAcls = conf.get(queuePath + aclApplicationMaxPriority, + conf.ALL_ACL); + + queuePriority = q.getPriority().getPriority(); + if (q instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + .getConfigName(); + } } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -220,6 +252,26 @@ public ResourceInfo getMaxEffectiveCapacity(){ return maxEffectiveCapacity; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public HashMap getAcls() { + return acls; + } + + public String getPriorityAcls() { + return priorityAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public boolean isLeafQueue() { return getQueues() == null; } -- 2.7.4 (Apple Git-66)