diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java index 28e261a..de9a254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java @@ -44,4 +44,12 @@ public static ReplaceLabelsOnNodeRequest newInstance( @Public @Evolving public abstract Map> getNodeToLabels(); + + @Public + @Evolving + public abstract void setCheckNodes(boolean checkNodes); + + @Public + @Evolving + public abstract boolean getCheckNodes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index eaf658f..f5fead1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -98,6 +98,7 @@ message RemoveFromClusterNodeLabelsResponseProto { message ReplaceLabelsOnNodeRequestProto { repeated NodeIdToLabelsNameProto nodeToLabels = 1; + optional bool checkNodes = 2; } message ReplaceLabelsOnNodeResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index d407c20..a1b89a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -124,11 +124,13 @@ new UsageInfo(" (label splitted by \",\")", "remove from cluster node labels")) .put("-replaceLabelsOnNode", - new UsageInfo( - "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\">", + new UsageInfo("[-checkNodes] " + + "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\">", "replace labels on nodes" + " (please note that we do not support specifying multiple" - + " labels on a single host for now.)")) + + " labels on a single host for now.)\n\t\t" + + "[-checkNodes] is optional, if we set this option, it will " + + "check existance of specified nodes.")) .put("-directlyAccessNodeLabelStore", new UsageInfo("", "This is DEPRECATED, will be removed in future releases. Directly access node label store, " + "with this option, all node label related operations" @@ -240,7 +242,7 @@ private static void printHelp(String cmd, boolean isHAEnabled) { " [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ]" + - " [-replaceLabelsOnNode <\"node1[:port]=label1,label2 node2[:port]=label1\">]" + + " [-replaceLabelsOnNode [-checkNodes] <\"node1[:port]=label1,label2 node2[:port]=label1\">]" + " [-directlyAccessNodeLabelStore]" + " [-updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout])"); if (isHAEnabled) { @@ -632,13 +634,13 @@ private int removeFromClusterNodeLabels(String args) throws IOException, return map; } - private int replaceLabelsOnNodes(String args) throws IOException, - YarnException { + private int replaceLabelsOnNodes(String args, boolean checkNodes) + throws IOException, YarnException { Map> map = buildNodeLabelsMapFromStr(args); - return replaceLabelsOnNodes(map); + return replaceLabelsOnNodes(map, checkNodes); } - private int replaceLabelsOnNodes(Map> map) + private int replaceLabelsOnNodes(Map> map, boolean checkNodes) throws IOException, YarnException { if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map); @@ -647,6 +649,7 @@ private int replaceLabelsOnNodes(Map> map) createAdminProtocol(); ReplaceLabelsOnNodeRequest request = ReplaceLabelsOnNodeRequest.newInstance(map); + request.setCheckNodes(checkNodes); adminProtocol.replaceLabelsOnNode(request); } return 0; @@ -787,8 +790,15 @@ public int run(String[] args) throws Exception { System.err.println(NO_MAPPING_ERR_MSG); printUsage("", isHAEnabled); exitCode = -1; + } else if(args.length == 3) { + if ("-checkNodes".equals(args[1])) { + exitCode = replaceLabelsOnNodes(args[i], true); + } else { + printUsage("", isHAEnabled); + return -1; + } } else { - exitCode = replaceLabelsOnNodes(args[i]); + exitCode = replaceLabelsOnNodes(args[i], false); } } else { exitCode = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java index 22e561c..7b63d73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsNameProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; @@ -146,6 +147,18 @@ public void setNodeToLabels(Map> map) { nodeIdToLabels.putAll(map); } + @Override + public void setCheckNodes(boolean checkNodes) { + maybeInitBuilder(); + builder.setCheckNodes(checkNodes); + } + + @Override + public boolean getCheckNodes() { + ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getCheckNodes(); + } + private NodeIdProto convertToProtoFormat(NodeId t) { return ((NodeIdPBImpl) t).getProto(); } @@ -165,4 +178,5 @@ public boolean equals(Object other) { } return false; } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index fbc2d6f..c5159b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -776,6 +776,19 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( ReplaceLabelsOnNodeResponse response = recordFactory.newRecordInstance(ReplaceLabelsOnNodeResponse.class); + // check node existence if necessary + if (request.getCheckNodes()) { + for (NodeId node : request.getNodeToLabels().keySet()) { + if (!rmContext.getRMNodes().containsKey(node)){ + RMAuditLogger.logFailure(user.getShortUserName(), operation, "", + "AdminService", "Replace labels on non exist nodes:" + + node.toString()); + throw RPCUtil.getRemoteException( + new IOException("Replace labels on non exist nodes:"+ node.toString())); + } + } + } + try { rmContext.getNodeLabelManager().replaceLabelsOnNode( request.getNodeToLabels()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 639b955..2b120fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -28,6 +28,7 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -47,7 +48,6 @@ import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -75,6 +76,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.io.Resources; public class TestRMAdminService { @@ -916,6 +918,62 @@ public void testModifyLabelsOnNodesWithCentralizedConfigurationDisabled() (Set) ImmutableSet.of("x")))); rm.close(); } + + @Test + public void testModifyLabelsOnNonExistNodes() + throws IOException, YarnException { + // create RM and set it's ACTIVE, and set distributed node label + // configuration to true + MockRM rm = new MockRM(); + + ((RMContextImpl) rm.getRMContext()) + .setHAServiceState(HAServiceState.ACTIVE); + Map rmNodes = rm.getRMContext().getRMNodes(); + rmNodes.put(NodeId.newInstance("host1", 0), + new RMNodeImpl(null, rm.getRMContext(), "host1", 0, 0, null, null, null)); + rmNodes.put(NodeId.newInstance("host2", 0), + new RMNodeImpl(null, rm.getRMContext(), "host2", 0, 0, null, null, null)); + rmNodes.put(NodeId.newInstance("host3", 0), + new RMNodeImpl(null, rm.getRMContext(), "host3", 0, 0, null, null, null)); + RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager(); + + // by default, distributed configuration for node label is disabled, this + + labelMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); + // replace existing node + ReplaceLabelsOnNodeRequest request1 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host1", 0), + (Set) ImmutableSet.of("x"))); + request1.setCheckNodes(true);; + try { + rm.adminService.replaceLabelsOnNode(request1); + } catch (Exception ex) { + fail("should not fail on existing node"); + } + + // replace non-exist node + ReplaceLabelsOnNodeRequest request2 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host4", 0), + (Set) ImmutableSet.of("x"))); + request2.setCheckNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request2); + fail("Should fail on non-existing node"); + } catch (Exception ex) { + } + + // replace non-exist node but not check + ReplaceLabelsOnNodeRequest request3 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host5", 0), + (Set) ImmutableSet.of("x"))); + request3.setCheckNodes(false); + try { + rm.adminService.replaceLabelsOnNode(request3); + } catch (Exception ex) { + fail("Should not fail on non-existing node when not check nodes"); + } + rm.close(); + } @Test public void testRemoveClusterNodeLabelsWithCentralizedConfigurationDisabled()