From d821f3556de40eb1d53dfffa250a26a993a68b8d Mon Sep 17 00:00:00 2001 From: prabhujoseph Date: Tue, 29 Oct 2019 12:24:37 +0530 Subject: [PATCH] YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Signed-off-by: prabhujoseph --- .../scheduler/capacity/CSQueue.java | 4 ++ .../webapp/JAXBContextResolver.java | 3 +- .../webapp/dao/CapacitySchedulerInfo.java | 11 +++-- .../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 30 ++++++++---- .../webapp/dao/CapacitySchedulerQueueInfo.java | 54 +++++++++++++++++++++- .../resourcemanager/webapp/dao/QueueAclInfo.java | 48 +++++++++++++++++++ .../resourcemanager/webapp/dao/QueueAclsInfo.java | 47 +++++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 4 +- 8 files changed, 183 insertions(+), 18 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index c0c280e..1af3250 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -86,6 +86,10 @@ public PrivilegedEntity getPrivilegedEntity(); + Resource getMaximumAllocation(); + + Resource getMinimumAllocation(); + /** * Get the configured capacity of the queue. * @return configured queue capacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2e4204e..8c9c056 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -55,7 +55,8 @@ public JAXBContextResolver() throws Exception { UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class, ResourceInformationsInfo.class }; + AppTimeoutInfo.class, ResourceInformationsInfo.class, + QueueAclsInfo.class, QueueAclInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 2498b29..8de7529 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -71,7 +71,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); - queues = getQueues(parent); + queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); @@ -140,7 +140,8 @@ public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues( + CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); // JAXB marashalling leads to situation where the "type" field injected @@ -163,10 +164,10 @@ protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + info = new CapacitySchedulerQueueInfo(cs, queue); + info.queues = getQueues(cs, queue); } queuesInfo.addToQueueInfoList(info); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 6176bef..2805352 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -45,6 +46,7 @@ protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected float configuredMaxAMResourceLimit; protected ResourceInfo AMResourceLimit; protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; @@ -53,9 +55,8 @@ protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; - - @XmlTransient - protected String orderingPolicyInfo; + protected long maxApplicationLifetime; + protected long defaultApplicationLifetime; @XmlTransient protected String orderingPolicyDisplayName; @@ -63,8 +64,8 @@ CapacitySchedulerLeafQueueInfo() { }; - CapacitySchedulerLeafQueueInfo(LeafQueue q) { - super(q); + CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) { + super(cs, q); numActiveApplications = q.getNumActiveApplications(); numPendingApplications = q.getNumPendingApplications(); numContainers = q.getNumContainers(); @@ -73,6 +74,7 @@ userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsersManager().getUsersInfo()); userLimitFactor = q.getUserLimitFactor(); + configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent(); AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); @@ -95,6 +97,8 @@ if ( q instanceof AutoCreatedLeafQueue) { isAutoCreatedLeafQueue = true; } + defaultApplicationLifetime = q.getDefaultApplicationLifetime(); + maxApplicationLifetime = q.getMaximumApplicationLifetime(); } @Override @@ -140,6 +144,10 @@ public UsersInfo getUsers() { public float getUserLimitFactor() { return userLimitFactor; } + + public float getConfiguredMaxAMResourceLimit() { + return configuredMaxAMResourceLimit; + } public ResourceInfo getAMResourceLimit() { return AMResourceLimit; @@ -165,10 +173,6 @@ public String getOrderingPolicyDisplayName() { return orderingPolicyDisplayName; } - public String getOrderingPolicyInfo() { - return orderingPolicyInfo; - } - public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; } @@ -180,4 +184,12 @@ public int getDefaultApplicationPriority() { public boolean isAutoCreatedLeafQueue() { return isAutoCreatedLeafQueue; } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index fd256b1..de1d9c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -27,10 +28,16 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -65,11 +72,15 @@ protected ResourcesInfo resources; protected ResourceInfo minEffectiveCapacity; protected ResourceInfo maxEffectiveCapacity; + protected ResourceInfo maximumAllocation; + protected QueueAclsInfo queueAcls; + protected int queuePriority; + protected String orderingPolicyInfo; CapacitySchedulerQueueInfo() { }; - CapacitySchedulerQueueInfo(CSQueue q) { + CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; @@ -114,6 +125,31 @@ q.getQueueResourceQuotas().getEffectiveMinResource()); maxEffectiveCapacity = new ResourceInfo( q.getQueueResourceQuotas().getEffectiveMaxResource()); + maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + queueAcls = new QueueAclsInfo(); + for (Map.Entry e : conf + .getAcls(queueName).entrySet()) { + QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(), + e.getValue().getAclString()); + queueAcls.add(queueAcl); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + String priorityAcls = conf.get(queuePath + aclApplicationMaxPriority, + conf.ALL_ACL); + + QueueAclInfo queueAcl = new QueueAclInfo( + AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls); + queueAcls.add(queueAcl); + + queuePriority = q.getPriority().getPriority(); + if (q instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + .getConfigName(); + } } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -220,6 +256,22 @@ public ResourceInfo getMaxEffectiveCapacity(){ return maxEffectiveCapacity; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public QueueAclsInfo getQueueAcls() { + return queueAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public boolean isLeafQueue() { return getQueues() == null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java new file mode 100644 index 0000000..8d78ed5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcl") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclInfo { + + protected String accessType; + protected String accessControlList; + + public QueueAclInfo() { + // JAXB needs this + } + + public QueueAclInfo(String accessType, String accessControlList) { + this.accessType = accessType; + this.accessControlList = accessControlList; + } + + public String getAccessType() { + return accessType; + } + + public String getAccessControlList() { + return accessControlList; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java new file mode 100644 index 0000000..38e836e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcls") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclsInfo { + + protected ArrayList queueAcl = new ArrayList(); + + public QueueAclsInfo() { + } // JAXB needs this + + public void add(QueueAclInfo queueAclInfo) { + queueAcl.add(queueAclInfo); + } + + public ArrayList getQueueAcls() { + return queueAcl; + } + + public void addAll(ArrayList queueAclsInfo) { + queueAcl.addAll(queueAclsInfo); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 8ab975f..6d9443e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -380,10 +380,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 20; + int numExpectedElements = 24; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 35; + numExpectedElements = 42; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); -- 2.7.4 (Apple Git-66)