diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 87c895a..a668c79 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -133,6 +133,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -715,6 +716,94 @@ public Response updateAppState(AppState targetState, return Response.status(Status.OK).entity(ret).build(); } + + @GET + @Path("/node-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException { + init(); + + NodeLabelsInfo ret = + new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager() + .getClusterNodeLabels()); + + return ret; + } + + @POST + @Path("/node-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response addLabels(final NodeLabelsInfo newNodeLabels, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User not authorized for this action " + + callerUGI.getShortUserName(); + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager() + .addToCluserNodeLabels(new HashSet( + newNodeLabels.getNodeLabels())); + + return Response.status(Status.OK).build(); + + } + + @GET + @Path("/nodes/{nodeId}/labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodeLabelsInfo getLabelsByNode(@Context HttpServletRequest hsr, + @PathParam("nodeId") String nodeId) + throws AuthorizationException, IOException { + init(); + + NodeId nid = ConverterUtils.toNodeId(nodeId); + return new NodeLabelsInfo( + rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid)); + + } + + @POST + @Path("/nodes/{nodeId}/labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response addLabels(NodeLabelsInfo newNodeLabelsInfo, + @Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User not authorized for this action " + + callerUGI.getShortUserName(); + throw new AuthorizationException(msg); + } + + NodeId nid = ConverterUtils.toNodeId(nodeId); + + Map> newLabelsForNode = new HashMap>(); + + newLabelsForNode.put(nid, new HashSet(newNodeLabelsInfo.getNodeLabels())); + + rm.getRMContext().getNodeLabelManager().addLabelsToNode(newLabelsForNode); + + return Response.status(Status.OK).build(); + + } protected Response killApp(RMApp app, UserGroupInformation callerUGI, HttpServletRequest hsr) throws IOException, InterruptedException { @@ -965,7 +1054,9 @@ protected ApplicationSubmissionContext createAppSubmissionContext( newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), - newApp.getKeepContainersAcrossApplicationAttempts()); + newApp.getKeepContainersAcrossApplicationAttempts(), + newApp.getAppLabelExpression(), + newApp.getAMContainerLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index f7233e6..d8d93e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -71,6 +71,12 @@ @XmlElementWrapper(name = "application-tags") @XmlElement(name = "tag") Set tags; + + @XmlElement(name = "app-label-expression") + String appLabelExpression; + + @XmlElement(name = "am-container-label-expression") + String amContainerLabelExpression; public ApplicationSubmissionContextInfo() { applicationId = ""; @@ -83,6 +89,8 @@ public ApplicationSubmissionContextInfo() { keepContainers = false; applicationType = ""; tags = new HashSet(); + appLabelExpression = ""; + amContainerLabelExpression = ""; } public String getApplicationId() { @@ -132,6 +140,14 @@ public boolean getKeepContainersAcrossApplicationAttempts() { public Set getApplicationTags() { return tags; } + + public String getAppLabelExpression() { + return appLabelExpression; + } + + public String getAMContainerLabelExpression() { + return amContainerLabelExpression; + } public void setApplicationId(String applicationId) { this.applicationId = applicationId; @@ -182,5 +198,12 @@ public void setApplicationType(String applicationType) { public void setApplicationTags(Set tags) { this.tags = tags; } + + public void setAppLabelExpression(String appLabelExpression) { + this.appLabelExpression = appLabelExpression; + } + public void setAMContainerLabelExpression(String labelExpression) { + this.amContainerLabelExpression = labelExpression; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index d5c84d8..7585717 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import java.util.Set; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -49,6 +51,8 @@ protected QueueState state; protected CapacitySchedulerQueueInfoList queues; protected ResourceInfo resourcesUsed; + protected float absActualCapacity; + protected NodeLabelsInfo labels = new NodeLabelsInfo(); private boolean hideReservationQueues = false; CapacitySchedulerQueueInfo() { @@ -71,6 +75,14 @@ queueName = q.getQueueName(); state = q.getState(); resourcesUsed = new ResourceInfo(q.getUsedResources()); + absActualCapacity = cap(q.getAbsActualCapacity(), 0f, 1f) * 100; + + // add labels + Set labelSet = q.getAccessibleNodeLabels(); + if (labelSet != null) { + labels = new NodeLabelsInfo(labelSet); + } + if(q instanceof PlanQueue && !((PlanQueue)q).showReservationsAsQueues()) { hideReservationQueues = true; @@ -100,6 +112,10 @@ public float getAbsoluteMaxCapacity() { public float getAbsoluteUsedCapacity() { return absoluteUsedCapacity; } + + public float getAbsActualCapacity() { + return absActualCapacity; + } public int getNumApplications() { return numApplications; @@ -127,6 +143,10 @@ public CapacitySchedulerQueueInfoList getQueues() { public ResourceInfo getResourcesUsed() { return resourcesUsed; } + + public NodeLabelsInfo getLabels() { + return labels; + } /** * Limit a value to a specified range. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 73a2db1..1eb8856 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -45,6 +49,7 @@ protected long availMemoryMB; protected long usedVirtualCores; protected long availableVirtualCores; + protected ArrayList labels = new ArrayList(); public NodeInfo() { } // JAXB needs this @@ -70,6 +75,13 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.lastHealthUpdate = ni.getLastHealthReportTime(); this.healthReport = String.valueOf(ni.getHealthReport()); this.version = ni.getNodeManagerVersion(); + + // add labels + Set labelSet = ni.getNodeLabels(); + if (labelSet != null) { + labels.addAll(labelSet); + Collections.sort(labels); + } } public String getRack() { @@ -123,5 +135,9 @@ public long getUsedVirtualCores() { public long getAvailableVirtualCores() { return this.availableVirtualCores; } + + public ArrayList getLabels() { + return this.labels; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java new file mode 100644 index 0000000..1cb895a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "nodeLabelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodeLabelsInfo { + + protected ArrayList nodeLabels = new ArrayList(); + + public NodeLabelsInfo() { + } // JAXB needs this + + public NodeLabelsInfo(ArrayList nodeLabels) { + this.nodeLabels = nodeLabels; + } + + public NodeLabelsInfo(Set nodeLabelsSet) { + this.nodeLabels = new ArrayList(nodeLabelsSet); + } + + public ArrayList getNodeLabels() { + return nodeLabels; + } + + public void setNodeLabels(ArrayList nodeLabels) { + this.nodeLabels = nodeLabels; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 0974311..bb38079 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -49,7 +49,7 @@ // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. final int numberOfThInMetricsTable = 16; - final int numberOfActualTableHeaders = 12; + final int numberOfActualTableHeaders = 13; private Injector injector; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index e58c30f..fc11cde 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -357,10 +357,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 12; + int numExpectedElements = 14; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 22; + numExpectedElements = 24; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());