diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index fcb9b74..9e7a481 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -41,10 +41,12 @@ import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.RMHAServiceTarget; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.resource.Resources; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -82,6 +85,7 @@ RecordFactoryProvider.getRecordFactory(null); private boolean directlyAccessNodeLabelStore = false; static CommonNodeLabelsManager localNodeLabelsManager = null; + private static YarnClient yarnClient = null; private static final String NO_LABEL_ERR_MSG = "No cluster node-labels are specified"; private static final String NO_MAPPING_ERR_MSG = @@ -91,6 +95,8 @@ private static final String ADD_LABEL_FORMAT_ERR_MSG = "Input format for adding node-labels is not correct, it should be " + "labelName1[(exclusive=true/false)],LabelName2[] .."; + private static final String REPLACE_LABEL_ON_UNKNOWN_NODES_MSG = + "Unknown nodes are specified. Nodes : "; protected final static Map ADMIN_USAGE = ImmutableMap.builder() @@ -130,11 +136,13 @@ new UsageInfo(" (label splitted by \",\")", "remove from cluster node labels")) .put("-replaceLabelsOnNode", - new UsageInfo( + new UsageInfo("[--fail-on-unkown-nodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\">", - "replace labels on nodes" - + " (please note that we do not support specifying multiple" - + " labels on a single host for now.)")) + "replace labels on nodes" + + " (please note that we do not support specifying multiple" + + " labels on a single host for now.)\n\t\t" + + "[--fail-on-unkown-nodes] is optional, when we set this" + + " option, it will fail if specified nodes are unknown.")) .put("-directlyAccessNodeLabelStore", new UsageInfo("", "This is DEPRECATED, will be removed in future releases. Directly access node label store, " + "with this option, all node label related operations" @@ -166,6 +174,11 @@ protected void setErrOut(PrintStream errOut) { this.errOut = errOut; } + @VisibleForTesting + protected static void setYarnClient(YarnClient client) { + yarnClient = client; + } + private static void appendHAUsage(final StringBuilder usageBuilder) { for (Map.Entry cmdEntry : USAGE.entrySet()) { if (cmdEntry.getKey().equals("-help") @@ -246,8 +259,8 @@ private static void printHelp(String cmd, boolean isHAEnabled) { " [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ]" + - " [-replaceLabelsOnNode <\"node1[:port]=label1,label2" + - " node2[:port]=label1\">]" + + " [-replaceLabelsOnNode [--fail-on-unkown-nodes] " + + "<\"node1[:port]=label1,label2 node2[:port]=label1\">]" + " [-directlyAccessNodeLabelStore]" + " [-updateNodeResource [NodeID] [MemSize] [vCores]" + " ([OvercommitTimeout])"); @@ -302,6 +315,14 @@ protected ResourceManagerAdministrationProtocol createAdminProtocol() ResourceManagerAdministrationProtocol.class); } + protected YarnClient createYarnClient() + throws IOException { + YarnClient client = YarnClient.createYarnClient(); + client.init(getConf()); + client.start(); + return client; + } + private int refreshQueues() throws IOException, YarnException { // Refresh the queue properties ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); @@ -656,14 +677,34 @@ private int removeFromClusterNodeLabels(String args) throws IOException, return map; } - private int replaceLabelsOnNodes(String args) throws IOException, - YarnException { + private int replaceLabelsOnNodes(String args, boolean verifyNodes) + throws IOException, YarnException { Map> map = buildNodeLabelsMapFromStr(args); - return replaceLabelsOnNodes(map); + return replaceLabelsOnNodes(map, verifyNodes); } - private int replaceLabelsOnNodes(Map> map) - throws IOException, YarnException { + private int replaceLabelsOnNodes(Map> map, + boolean verifyNodes) throws IOException, YarnException { + // with option "--fail-on-unknown-nodes", we will check node existence + // before replace labels on nodes. + if (verifyNodes) { + if (yarnClient == null) { + yarnClient = createYarnClient(); + } + List nodeReports = yarnClient.getNodeReports(); + List unknownNodes = new ArrayList(); + // Verify if requested nodes are known + for (NodeId requestedNode : map.keySet()) { + if (!isRMRegisteredNode(requestedNode, nodeReports)) { + unknownNodes.add(requestedNode); + } + } + if (!unknownNodes.isEmpty()) { + System.err.println(REPLACE_LABEL_ON_UNKNOWN_NODES_MSG + + unknownNodes.toString()); + return -1; + } + } if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map); } else { @@ -675,7 +716,30 @@ private int replaceLabelsOnNodes(Map> map) } return 0; } - + + private boolean acceptNode(NodeId requestedNode, NodeId knownNode) { + if (!requestedNode.getHost().equals(knownNode.getHost())) { + return false; + } else if (requestedNode.getPort() != 0) { + // requested node port is specified + return requestedNode.getPort() == knownNode.getPort(); + } else { + // requested node port is not specified + return true; + } + } + + private boolean isRMRegisteredNode(NodeId requestedNode, + List registeredNodes) { + for (NodeReport nodeReport : registeredNodes) { + NodeId knownNode = nodeReport.getNodeId(); + if (acceptNode(requestedNode, knownNode)) { + return true; + } + } + return false; + } + @Override public int run(String[] args) throws Exception { // -directlyAccessNodeLabelStore is a additional option for node label @@ -782,8 +846,15 @@ public int run(String[] args) throws Exception { System.err.println(NO_MAPPING_ERR_MSG); printUsage("", isHAEnabled); exitCode = -1; + } else if(args.length == 3) { + if ("--fail-on-unkown-nodes".equals(args[1])) { + exitCode = replaceLabelsOnNodes(args[2], true); + } else { + printUsage("", isHAEnabled); + return -1; + } } else { - exitCode = replaceLabelsOnNodes(args[i]); + exitCode = replaceLabelsOnNodes(args[i], false); } } else { exitCode = -1; @@ -817,6 +888,9 @@ public int run(String[] args) throws Exception { if (null != localNodeLabelsManager) { localNodeLabelsManager.stop(); } + if (null != yarnClient) { + yarnClient.stop(); + } return exitCode; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 60c7eac..4962d89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -36,8 +36,10 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; @@ -48,8 +50,11 @@ import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; +import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -67,7 +72,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; @@ -87,6 +91,7 @@ private RMAdminCLI rmAdminCLI; private RMAdminCLI rmAdminCLIWithHAEnabled; private CommonNodeLabelsManager dummyNodeLabelsManager; + private YarnClientImpl dummyYarnClient; private boolean remoteAdminServiceAccessed = false; @SuppressWarnings("static-access") @@ -125,7 +130,9 @@ protected HAServiceTarget resolveTarget(String rmId) { } }; initDummyNodeLabelsManager(); + initDummyYarnClient(); rmAdminCLI.localNodeLabelsManager = dummyNodeLabelsManager; + rmAdminCLI.setYarnClient(dummyYarnClient); YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); @@ -164,7 +171,33 @@ public void replaceLabelsOnNode( }; dummyNodeLabelsManager.init(conf); } - + + private void initDummyYarnClient() { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + dummyYarnClient = new YarnClientImpl() { + @Override + public List getNodeReports(NodeState... states) + throws YarnException, IOException { + NodeReport nodeReport1 = NodeReport.newInstance( + NodeId.newInstance("node1", 45454), NodeState.RUNNING, null, null, + null, null, 0, null, 0); + NodeReport nodeReport2 = NodeReport.newInstance( + NodeId.newInstance("node2", 45454), NodeState.RUNNING, null, null, + null, null, 0, null, 0); + NodeReport nodeReport3 = NodeReport.newInstance( + NodeId.newInstance("node3", 45454), NodeState.UNHEALTHY, null, + null, null, null, 0, null, 0); + List reports = new ArrayList(); + reports.add(nodeReport1); + reports.add(nodeReport2); + reports.add(nodeReport3); + return reports; + } + }; + dummyYarnClient.init(conf); + } + @Test(timeout=500) public void testRefreshQueues() throws Exception { String[] args = { "-refreshQueues" }; @@ -469,7 +502,7 @@ public void testHelp() throws Exception { "[username]] [-addToClusterNodeLabels " + "<\"label1(exclusive=true),label2(exclusive=false),label3\">] " + "[-removeFromClusterNodeLabels ] " + - "[-replaceLabelsOnNode " + + "[-replaceLabelsOnNode [--fail-on-unkown-nodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1\">] " + "[-directlyAccessNodeLabelStore] [-updateNodeResource " + "[NodeID] [MemSize] [vCores] ([OvercommitTimeout]) " + @@ -563,7 +596,8 @@ public void testHelp() throws Exception { + "[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup" + " [username]] [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" - + " [-removeFromClusterNodeLabels ] [-replaceLabelsOnNode " + + " [-removeFromClusterNodeLabels ] " + + "[-replaceLabelsOnNode [--fail-on-unkown-nodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1\">] [-directlyAccessNodeLabelStore] " + "[-updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout]) " + "[-transitionToActive [--forceactive] ] " @@ -790,6 +824,37 @@ public void testReplaceMultipleLabelsOnSingleNode() throws Exception { } @Test + public void testReplaceLabelsWithFailOnUnknownNodesOption() + throws Exception { + dummyNodeLabelsManager.addToCluserNodeLabelsWithDefaultExclusivity( + ImmutableSet.of("x", "y")); + + // known hosts are: + // node1:45454 + // node2:45454 + // node3:45454 + // should succeed, known nodes + String[] args1 = {"-replaceLabelsOnNode", "--fail-on-unkown-nodes", + "node1:45454=x node2:45454=y", "-directlyAccessNodeLabelStore"}; + assertTrue(0 == rmAdminCLI.run(args1)); + + // should fail, unknown nodes + String[] args2 = {"-replaceLabelsOnNode", "--fail-on-unkown-nodes", + "node1:45454=x node4:45454=y", "-directlyAccessNodeLabelStore"}; + assertTrue(0 != rmAdminCLI.run(args2)); + + // should succeed, known nodes with omitted port + String[] args3 = {"-replaceLabelsOnNode", "--fail-on-unkown-nodes", + "node1=x node2=y", "-directlyAccessNodeLabelStore"}; + assertTrue(0 == rmAdminCLI.run(args3)); + + // should fail, known nodes with wrong port + String[] args4 = {"-replaceLabelsOnNode", "--fail-on-unkown-nodes", + "node1:5555=x node2:6666=y", "-directlyAccessNodeLabelStore"}; + assertTrue(0 != rmAdminCLI.run(args4)); + } + + @Test public void testRemoveLabelsOnNodes() throws Exception { // Successfully replace labels dummyNodeLabelsManager