diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 640f8e3..4011754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -29,6 +29,12 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -80,7 +86,6 @@ private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - private boolean directlyAccessNodeLabelStore = false; static CommonNodeLabelsManager localNodeLabelsManager = null; private static final String NO_LABEL_ERR_MSG = "No cluster node-labels are specified"; @@ -91,6 +96,8 @@ private static final String ADD_LABEL_FORMAT_ERR_MSG = "Input format for adding node-labels is not correct, it should be " + "labelName1[(exclusive=true/false)],LabelName2[] .."; + private static final String INVALID_ARGS_ERR_MSG = + "Invalid args specified: "; protected final static Map ADMIN_USAGE = ImmutableMap.builder() @@ -576,11 +583,33 @@ private int getGroups(String[] usernames) throws IOException { return labels; } - private int addToClusterNodeLabels(String args) throws IOException, - YarnException { - List labels = buildNodeLabelsFromStr(args); + private int handleAddToClusterNodeLabels(String[] args, String cmd, + boolean isHAEnabled) throws IOException, YarnException { + Options opts = new Options(); + opts.addOption("addToClusterNodeLabels", false, + "Add to cluster node labels."); + opts.addOption("directlyAccessNodeLabelStore", false, + "Directly access node label store."); + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (ParseException ex) { + printUsage(args[0], isHAEnabled); + return exitCode; + } + + String[] remainArgs = cliParser.getArgs(); + if (remainArgs.length < 1){ + System.err.println(NO_LABEL_ERR_MSG); + printUsage(args[0], isHAEnabled); + } else if (remainArgs.length > 1) { + System.err.println(INVALID_ARGS_ERR_MSG + Arrays.toString(remainArgs)); + printUsage(args[0], isHAEnabled); + } - if (directlyAccessNodeLabelStore) { + List labels = buildNodeLabelsFromStr(remainArgs[0]); + if (cliParser.hasOption("directlyAccessNodeLabelStore")) { getNodeLabelManagerInstance(getConf()).addToCluserNodeLabels(labels); } else { ResourceManagerAdministrationProtocol adminProtocol = @@ -592,11 +621,33 @@ private int addToClusterNodeLabels(String args) throws IOException, return 0; } - private int removeFromClusterNodeLabels(String args) throws IOException, - YarnException { - Set labels = buildNodeLabelNamesFromStr(args); + private int handleRemoveFromClusterNodeLabels(String[] args, String cmd, + boolean isHAEnabled) throws IOException, YarnException { + Options opts = new Options(); + opts.addOption("removeFromClusterNodeLabels", false, + "Remove From cluster node labels."); + opts.addOption("directlyAccessNodeLabelStore", false, + "Directly access node label store."); + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (ParseException ex) { + printUsage(args[0], isHAEnabled); + return exitCode; + } - if (directlyAccessNodeLabelStore) { + String[] remainArgs = cliParser.getArgs(); + if (remainArgs.length < 1){ + System.err.println(NO_LABEL_ERR_MSG); + printUsage(args[0], isHAEnabled); + } else if (remainArgs.length > 1) { + System.err.println(INVALID_ARGS_ERR_MSG + Arrays.toString(remainArgs)); + printUsage(args[0], isHAEnabled); + } + + Set labels = buildNodeLabelNamesFromStr(remainArgs[0]); + if (cliParser.hasOption("directlyAccessNodeLabelStore")) { getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels( labels); } else { @@ -659,15 +710,43 @@ private int removeFromClusterNodeLabels(String args) throws IOException, return map; } - private int replaceLabelsOnNodes(String args, boolean failOnUnknownNodes) - throws IOException, YarnException { - Map> map = buildNodeLabelsMapFromStr(args); - return replaceLabelsOnNodes(map, failOnUnknownNodes); + private int handleReplaceLabelsOnNodes(String[] args, String cmd, + boolean isHAEnabled) throws IOException, YarnException { + Options opts = new Options(); + opts.addOption("replaceLabelsOnNode", false, + "Replace label on node."); + opts.addOption("failOnUnknownNodes", false, + "Fail on unknown nodes."); + opts.addOption("directlyAccessNodeLabelStore", false, + "Directly access node label store."); + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (ParseException ex) { + printUsage(args[0], isHAEnabled); + return exitCode; + } + + String[] remainArgs = cliParser.getArgs(); + if (remainArgs.length < 1){ + System.err.println(NO_MAPPING_ERR_MSG); + printUsage(args[0], isHAEnabled); + } else if (remainArgs.length > 1) { + System.err.println(INVALID_ARGS_ERR_MSG + Arrays.toString(remainArgs)); + printUsage(args[0], isHAEnabled); + } + + Map> map = buildNodeLabelsMapFromStr(remainArgs[0]); + return replaceLabelsOnNodes(map, + cliParser.hasOption("failOnUnknownNodes"), + cliParser.hasOption("directlyAccessNodeLabelStore")); } private int replaceLabelsOnNodes(Map> map, - boolean failOnUnknownNodes) throws IOException, YarnException { - if (directlyAccessNodeLabelStore) { + boolean failOnUnknownNodes, boolean directlyAccessNodeLabelStore) + throws IOException, YarnException { + if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map); } else { ResourceManagerAdministrationProtocol adminProtocol = @@ -682,18 +761,6 @@ private int replaceLabelsOnNodes(Map> map, @Override public int run(String[] args) throws Exception { - // -directlyAccessNodeLabelStore is a additional option for node label - // access, so just search if we have specified this option, and remove it - List argsList = new ArrayList(); - for (int i = 0; i < args.length; i++) { - if (args[i].equals("-directlyAccessNodeLabelStore")) { - directlyAccessNodeLabelStore = true; - } else { - argsList.add(args[i]); - } - } - args = argsList.toArray(new String[0]); - YarnConfiguration yarnConf = getConf() == null ? new YarnConfiguration() : new YarnConfiguration( getConf()); @@ -766,37 +833,11 @@ public int run(String[] args) throws Exception { } else if ("-updateNodeResource".equals(cmd)) { exitCode = handleUpdateNodeResource(args, cmd, isHAEnabled); } else if ("-addToClusterNodeLabels".equals(cmd)) { - if (i >= args.length) { - System.err.println(NO_LABEL_ERR_MSG); - printUsage("", isHAEnabled); - exitCode = -1; - } else { - exitCode = addToClusterNodeLabels(args[i]); - } + exitCode = handleAddToClusterNodeLabels(args, cmd, isHAEnabled); } else if ("-removeFromClusterNodeLabels".equals(cmd)) { - if (i >= args.length) { - System.err.println(NO_LABEL_ERR_MSG); - printUsage("", isHAEnabled); - exitCode = -1; - } else { - exitCode = removeFromClusterNodeLabels(args[i]); - } + exitCode = handleRemoveFromClusterNodeLabels(args, cmd, isHAEnabled); } else if ("-replaceLabelsOnNode".equals(cmd)) { - if (i >= args.length) { - System.err.println(NO_MAPPING_ERR_MSG); - printUsage("", isHAEnabled); - exitCode = -1; - } else if ("-failOnUnknownNodes".equals(args[i])) { - if (i + 1 >= args.length) { - System.err.println(NO_MAPPING_ERR_MSG); - printUsage("", isHAEnabled); - exitCode = -1; - } else { - exitCode = replaceLabelsOnNodes(args[i + 1], true); - } - } else { - exitCode = replaceLabelsOnNodes(args[i], false); - } + exitCode = handleReplaceLabelsOnNodes(args, cmd, isHAEnabled); } else { exitCode = -1; System.err.println(cmd.substring(1) + ": Unknown command"); @@ -835,27 +876,49 @@ public int run(String[] args) throws Exception { // A helper method to reduce the number of lines of run() private int handleRefreshNodes(String[] args, String cmd, boolean isHAEnabled) throws IOException, YarnException { - if (args.length == 1) { - return refreshNodes(); - } else if (args.length == 3 || args.length == 4) { - // if the graceful timeout specified - if ("-g".equals(args[1]) || "-graceful".equals(args[1])) { - int timeout = -1; - String trackingMode; - if (args.length == 4) { - timeout = validateTimeout(args[2]); - trackingMode = validateTrackingMode(args[3]); - } else { - trackingMode = validateTrackingMode(args[2]); - } - return refreshNodes(timeout, trackingMode); + Options opts = new Options(); + opts.addOption("refreshNodes", false, + "Refresh the hosts information at the ResourceManager."); + Option gracefulOpt = new Option("g", "graceful", true, + "Wait for timeout before marking the NodeManager as decommissioned."); + gracefulOpt.setOptionalArg(true); + opts.addOption(gracefulOpt); + opts.addOption("client", false, + "Indicates the timeout tracking should be handled by the client."); + opts.addOption("server", false, + "Indicates the timeout tracking should be handled by the RM."); + + int exitCode = -1; + CommandLine cliParser = null; + try { + cliParser = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + System.out.println("Missing argument for options"); + printUsage(args[0], isHAEnabled); + return exitCode; + } catch (ParseException ex) { + printUsage(args[0], isHAEnabled); + return exitCode; + } + + int timeout = -1; + if (cliParser.hasOption("g")) { + String strTimeout = cliParser.getOptionValue("g"); + if (strTimeout != null) { + timeout = validateTimeout(strTimeout); + } + String trackingMode = null; + if (cliParser.hasOption("client")) { + trackingMode = "client"; + } else if (cliParser.hasOption("server")) { + trackingMode = "server"; } else { printUsage(cmd, isHAEnabled); return -1; } + return refreshNodes(timeout, trackingMode); } else { - printUsage(cmd, isHAEnabled); - return -1; + return refreshNodes(); } } @@ -894,16 +957,6 @@ private int validateTimeout(String strTimeout) { return timeout; } - private String validateTrackingMode(String mode) { - if ("-client".equals(mode)) { - return "client"; - } - if ("-server".equals(mode)) { - return "server"; - } - throw new IllegalArgumentException("Invalid mode specified: " + mode); - } - @Override public void setConf(Configuration conf) { if (conf != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 9e20a43..ad0ac1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -67,7 +67,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; @@ -780,6 +779,19 @@ public void testReplaceLabelsOnNode() throws Exception { args = new String[] { "-replaceLabelsOnNode", ", " }; assertTrue(0 != rmAdminCLI.run(args)); + + // no labels but have option, should fail + args = new String[] { "-replaceLabelsOnNode", "-failOnUnknownNodes" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // change the sequence of labels and options, + // should not matter + args = new String[] + { "-replaceLabelsOnNode", + "-failOnUnknownNodes", + "node1:8000,x node2:8000=y node3,x node4=Y", + "-directlyAccessNodeLabelStore" }; + assertEquals(0, rmAdminCLI.run(args)); } @Test