diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 24a90bd..1411e1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -132,6 +132,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelNamesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -714,6 +719,140 @@ public Response updateAppState(AppState targetState, return Response.status(Status.OK).entity(ret).build(); } + + @GET + @Path("/labels/all-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public LabelNamesInfo getLabels(@Context HttpServletRequest hsr) throws AuthorizationException, IOException { + init(); + + LabelNamesInfo ret = new LabelNamesInfo(rm.getRMContext().getNodeLabelManager().getLabels()); + + return ret; + } + + @GET + @Path("/labels/all-nodes-to-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NodesToLabelsInfo getNodesToLabels(@Context HttpServletRequest hsr, + @QueryParam("labels") Set labelsQuery) throws AuthorizationException, IOException { + init(); + + NodesToLabelsInfo nodesToLabelsInfo = new NodesToLabelsInfo(); + + Map> nodesToLabels = rm.getRMContext().getNodeLabelManager().getNodesToLabels(); + + boolean filterLabels = false; + if (labelsQuery != null && !labelsQuery.isEmpty()) { + filterLabels = true; + } + + for (Map.Entry> nlEntry : nodesToLabels.entrySet()) { + Set nodeLabels = nlEntry.getValue(); + if (filterLabels) { + Set labelIntersect = new HashSet(nodeLabels); + labelIntersect.retainAll(labelsQuery); + if (!labelIntersect.isEmpty()) { + nodesToLabelsInfo.add(new NodeToLabelsInfo(nlEntry.getKey(), labelIntersect)); + } + } else { + nodesToLabelsInfo.add(new NodeToLabelsInfo(nlEntry.getKey(), nlEntry.getValue())); + } + } + + return nodesToLabelsInfo; + } + + @POST + @Path("/labels/add-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response addLabels(final LabelNamesInfo newLabels, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User not authorized for this action " + + callerUGI.getShortUserName(); + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager() + .addLabels(new HashSet(newLabels.getLabels())); + + return Response.status(Status.OK).build(); + + } + + @POST + @Path("/labels/remove-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response removeLabels(final LabelNamesInfo oldLabels, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User not authorized for this action " + + callerUGI.getShortUserName(); + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager() + .removeLabels(new HashSet(oldLabels.getLabels())); + + return Response.status(Status.OK).build(); + + } + + @POST + @Path("/labels/set-node-to-labels") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response addLabels(NodesToLabelsInfo newNodesToLabelsInfo, + @Context HttpServletRequest hsr) + throws Exception { + init(); + + final Map> newNodeToLabels = new HashMap>(); + + for (NodeToLabelsInfo nodeToLabelsInfo : newNodesToLabelsInfo.getNodeToLabelsInfos()) { + //It's a list, the same node could be specified > once + Set labels = newNodeToLabels.get(nodeToLabelsInfo.getNode()); + if (labels == null) { + labels = new HashSet(); + newNodeToLabels.put(nodeToLabelsInfo.getNode(), labels); + } + labels.addAll(nodeToLabelsInfo.getLabels()); + } + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + + if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) { + String msg = "User not authorized for this action " + + callerUGI.getShortUserName(); + throw new AuthorizationException(msg); + } + + rm.getRMContext().getNodeLabelManager().setLabelsOnMultipleNodes(newNodeToLabels); + + + return Response.status(Status.OK).build(); + + } protected Response killApp(RMApp app, UserGroupInformation callerUGI, HttpServletRequest hsr) throws IOException, InterruptedException { @@ -964,7 +1103,9 @@ protected ApplicationSubmissionContext createAppSubmissionContext( newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), - newApp.getKeepContainersAcrossApplicationAttempts()); + newApp.getKeepContainersAcrossApplicationAttempts(), + newApp.getAppLabelExpression(), + newApp.getAMContainerLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index f7233e6..d8d93e5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -71,6 +71,12 @@ @XmlElementWrapper(name = "application-tags") @XmlElement(name = "tag") Set tags; + + @XmlElement(name = "app-label-expression") + String appLabelExpression; + + @XmlElement(name = "am-container-label-expression") + String amContainerLabelExpression; public ApplicationSubmissionContextInfo() { applicationId = ""; @@ -83,6 +89,8 @@ public ApplicationSubmissionContextInfo() { keepContainers = false; applicationType = ""; tags = new HashSet(); + appLabelExpression = ""; + amContainerLabelExpression = ""; } public String getApplicationId() { @@ -132,6 +140,14 @@ public boolean getKeepContainersAcrossApplicationAttempts() { public Set getApplicationTags() { return tags; } + + public String getAppLabelExpression() { + return appLabelExpression; + } + + public String getAMContainerLabelExpression() { + return amContainerLabelExpression; + } public void setApplicationId(String applicationId) { this.applicationId = applicationId; @@ -182,5 +198,12 @@ public void setApplicationType(String applicationType) { public void setApplicationTags(Set tags) { this.tags = tags; } + + public void setAppLabelExpression(String appLabelExpression) { + this.appLabelExpression = appLabelExpression; + } + public void setAMContainerLabelExpression(String labelExpression) { + this.amContainerLabelExpression = labelExpression; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index ac16ce0..4f38d2d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -48,6 +52,8 @@ protected QueueState state; protected CapacitySchedulerQueueInfoList queues; protected ResourceInfo resourcesUsed; + protected float absActualCapacity; + protected ArrayList labels = new ArrayList(); CapacitySchedulerQueueInfo() { }; @@ -69,6 +75,14 @@ queueName = q.getQueueName(); state = q.getState(); resourcesUsed = new ResourceInfo(q.getUsedResources()); + absActualCapacity = cap(q.getAbsActualCapacity(), 0f, 1f) * 100; + + // add labels + Set labelSet = q.getLabels(); + if (labelSet != null) { + labels.addAll(labelSet); + Collections.sort(labels); + } } public float getCapacity() { @@ -94,6 +108,10 @@ public float getAbsoluteMaxCapacity() { public float getAbsoluteUsedCapacity() { return absoluteUsedCapacity; } + + public float getAbsActualCapacity() { + return absActualCapacity; + } public int getNumApplications() { return numApplications; @@ -118,6 +136,10 @@ public CapacitySchedulerQueueInfoList getQueues() { public ResourceInfo getResourcesUsed() { return resourcesUsed; } + + public ArrayList getLabels() { + return labels; + } /** * Limit a value to a specified range. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelInfo.java new file mode 100644 index 0000000..bd1f926 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelInfo.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "labelInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class LabelInfo { + + protected String labelName; + protected ArrayList activeNodes = new ArrayList(); + protected ArrayList inactiveNodes = new ArrayList(); + + public LabelInfo() { + } // JAXB needs this + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelNamesInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelNamesInfo.java new file mode 100644 index 0000000..e71954d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelNamesInfo.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "labelNamesInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class LabelNamesInfo { + + protected ArrayList labels = new ArrayList(); + + public LabelNamesInfo() { + } // JAXB needs this + + public LabelNamesInfo(Set labelSet) { + labels.addAll(labelSet); + //labels.add("foo"); + } + + public ArrayList getLabels() { + return labels; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsInfo.java new file mode 100644 index 0000000..481fe40 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsInfo.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "labelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class LabelsInfo { + + protected ArrayList labels = new ArrayList(); + + public LabelsInfo() { + } // JAXB needs this + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 73a2db1..bdfc6dd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -45,6 +49,7 @@ protected long availMemoryMB; protected long usedVirtualCores; protected long availableVirtualCores; + protected ArrayList labels = new ArrayList(); public NodeInfo() { } // JAXB needs this @@ -70,6 +75,13 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.lastHealthUpdate = ni.getLastHealthReportTime(); this.healthReport = String.valueOf(ni.getHealthReport()); this.version = ni.getNodeManagerVersion(); + + // add labels + Set labelSet = ni.getLabels(); + if (labelSet != null) { + labels.addAll(labelSet); + Collections.sort(labels); + } } public String getRack() { @@ -123,5 +135,9 @@ public long getUsedVirtualCores() { public long getAvailableVirtualCores() { return this.availableVirtualCores; } + + public ArrayList getLabels() { + return this.labels; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java new file mode 100644 index 0000000..527d12c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "nodeToLabelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodeToLabelsInfo { + + protected String node; + protected ArrayList labels = new ArrayList(); + + public NodeToLabelsInfo() { + } // JAXB needs this + + public NodeToLabelsInfo(String node) { + this.node = node; + } + + public NodeToLabelsInfo(String node, Set labels) { + this.node = node; + this.labels.addAll(labels); + } + + public String getNode() { + return node; + } + + public ArrayList getLabels() { + return labels; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesToLabelsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesToLabelsInfo.java new file mode 100644 index 0000000..3c2eb08 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesToLabelsInfo.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.*; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "nodesToLabelsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class NodesToLabelsInfo { + + protected ArrayList nodeToLabelsInfos = new ArrayList(); + + public NodesToLabelsInfo() { + } // JAXB needs this + + public ArrayList getNodeToLabelsInfos() { + return nodeToLabelsInfos; + } + + public void add(NodeToLabelsInfo nodeToLabelInfo) { + nodeToLabelsInfos.add(nodeToLabelInfo); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java new file mode 100644 index 0000000..bfff677 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -0,0 +1,376 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesToLabelsInfo; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.json.JSONJAXBContext; +import com.sun.jersey.api.json.JSONMarshaller; +import com.sun.jersey.api.json.JSONUnmarshaller; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; + +public class TestRMWebServicesNodeLabels extends JerseyTest { + + private static final Log LOG = LogFactory.getLog(TestRMWebServicesNodeLabels.class); + + private static MockRM rm; + private YarnConfiguration conf; + + private String userName; + private String notUserName; + + private Injector injector = Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + try { + userName = UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException ioe) { + throw new RuntimeException("Unable to get current user name " + ioe.getMessage(), ioe); + } + notUserName = userName + "abc123"; + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + filter("/*").through(TestRMWebServicesAppsModification.TestRMCustomAuthFilter.class); + serve("/*").with(GuiceContainer.class); + } + }); + + public class GuiceServletConfig extends GuiceServletContextListener { + + @Override + protected Injector getInjector() { + return injector; + } + } + + public TestRMWebServicesNodeLabels() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + + @Test + public void testNodeLabels() throws JSONException, Exception { + WebResource r = resource(); + + ClientResponse response; + JSONObject json; + JSONArray jarr; + String responseString; + + //Add a label + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("add-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"a\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + //Verify it is present + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("labels")); + + //Add another + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("add-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"b\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + + //Verify + jarr = json.getJSONArray("labels"); + assertEquals(2, jarr.length()); + + //Remove one + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("remove-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"a\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + //Verify + assertEquals("b", json.getString("labels")); + + //Add a node->label mapping + NodesToLabelsInfo nsli = new NodesToLabelsInfo(); + NodeToLabelsInfo nli = new NodeToLabelsInfo("node1"); + nli.getLabels().add("b"); + nsli.add(nli); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("set-node-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(nsli, NodesToLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + //Verify + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(1, nsli.getNodeToLabelsInfos().size()); + nli = nsli.getNodeToLabelsInfos().get(0); + assertEquals("node1", nli.getNode()); + assertEquals(1, nli.getLabels().size()); + assertTrue(nli.getLabels().contains("b")); + + //Get with filter which should suppress results + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels").queryParam("labels", "a") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(0, nsli.getNodeToLabelsInfos().size()); + + //Get with filter which should include results + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels").queryParam("labels", "b") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(1, nsli.getNodeToLabelsInfos().size()); + + //"Remove" by setting with an empty label set + nli = nsli.getNodeToLabelsInfos().get(0); + nli.getLabels().remove("b"); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("set-node-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(nsli, NodesToLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(1, nsli.getNodeToLabelsInfos().size()); + nli = nsli.getNodeToLabelsInfos().get(0); + assertTrue(nli.getLabels().isEmpty()); + } + + @Test + public void testNodeLabelsAuthFail() throws JSONException, Exception { + + + WebResource r = resource(); + + ClientResponse response; + JSONObject json; + JSONArray jarr; + String responseString; + + //Add a label + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("add-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"a\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + //Verify it is present + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("a", json.getString("labels")); + + //Fail adding another + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("add-labels") + .queryParam("user.name", notUserName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"b\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + + //Verify + assertEquals("a", json.getString("labels")); + + //Faile to remove one + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("remove-labels") + .queryParam("user.name", notUserName) + .accept(MediaType.APPLICATION_JSON) + .entity("{\"labels\":\"a\"}", MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + //Verify + assertEquals("a", json.getString("labels")); + + //Add a node->label mapping + NodesToLabelsInfo nsli = new NodesToLabelsInfo(); + NodeToLabelsInfo nli = new NodeToLabelsInfo("node1"); + nli.getLabels().add("a"); + nsli.add(nli); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("set-node-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(nsli, NodesToLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + //Verify + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(1, nsli.getNodeToLabelsInfos().size()); + nli = nsli.getNodeToLabelsInfos().get(0); + assertEquals("node1", nli.getNode()); + assertEquals(1, nli.getLabels().size()); + assertTrue(nli.getLabels().contains("a")); + + //Fail "Remove" by setting with an empty label set + nli = nsli.getNodeToLabelsInfos().get(0); + nli.getLabels().remove("a"); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("set-node-to-labels") + .queryParam("user.name", notUserName) + .accept(MediaType.APPLICATION_JSON) + .entity(toJson(nsli, NodesToLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); + + response = r.path("ws").path("v1").path("cluster") + .path("labels").path("all-nodes-to-labels") + .queryParam("user.name", userName) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + responseString = response.getEntity(String.class); + LOG.info(responseString); + nsli = (NodesToLabelsInfo) fromJson(responseString, NodesToLabelsInfo.class); + assertEquals(1, nsli.getNodeToLabelsInfos().size()); + nli = nsli.getNodeToLabelsInfos().get(0); + assertFalse(nli.getLabels().isEmpty()); + + } + + private String toJson(Object nsli, Class klass) throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(nsli, sw); + return sw.toString(); + } + + private Object fromJson(String json, Class klass) throws Exception { + StringReader sr = new StringReader(json); + JSONJAXBContext ctx = new JSONJAXBContext(klass); + JSONUnmarshaller jm = ctx.createJSONUnmarshaller(); + return jm.unmarshalFromJSON(sr, klass); + } + +}