diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index bfcb190..edfdc5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -28,6 +28,7 @@ function hadoop_usage echo " jar run a jar file" echo " logs dump container logs" echo " node prints node report(s)" + echo " node-labels prints node-label information(s)" echo " nodemanager run a nodemanager on each slave" echo " proxyserver run the web app proxy server" echo " resourcemanager run the ResourceManager" @@ -109,6 +110,11 @@ case "${COMMAND}" in hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS" YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" ;; + node-labels) + CLASS=org.apache.hadoop.yarn.client.cli.NodeLabelsCLI + hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS" + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; nodemanager) daemon="true" CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 0cf0b40..7b0019f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -196,6 +196,11 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof +:node-labels + set CLASS=org.apache.hadoop.yarn.client.cli.NodeLabelsCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager @@ -298,6 +303,7 @@ goto :eof @echo applicationattempt prints applicationattempt(s) report @echo container prints container(s) report @echo node prints node report(s) + @echo node-labels prints node-label information(s) @echo logs dump container logs @echo classpath prints the class path needed to get the @echo Hadoop jar and the required libraries diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java new file mode 100644 index 0000000..8eeaea0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeLabelsCLI.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +import com.google.common.annotations.VisibleForTesting; + +/** + * YARN Client used to get labels and node-to-label mappings of the cluster + */ +public class NodeLabelsCLI extends YarnCLI { + private static final Log LOG = LogFactory.getLog(NodeLabelsCLI.class); + + private static final String NODE_ID_CMD = "nodeId"; + private static final String TITLE = "yarn node-labels"; + private static final String LIST_LABELS_CMD = "listLabels"; + private static final String LIST_MAPPINGS_CMD = "listMappings"; + private static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = + "directlyAccessNodeLabelStore"; + private boolean accessLocal = false; + static CommonNodeLabelsManager localNodeLabelsManager = null; + + public static void main(String[] args) throws Exception { + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + private void validateOptionsAndThrowExceptionIfInvalid(CommandLine parsedCli) { + boolean hasListLabels = parsedCli.hasOption(LIST_LABELS_CMD); + boolean hasListMappings = parsedCli.hasOption(LIST_MAPPINGS_CMD); + // user cannot specify list labels and list mappings at the same time + if (hasListLabels && hasListMappings) { + String msg = + "-" + LIST_LABELS_CMD + " and -" + LIST_MAPPINGS_CMD + + " cannot be specifed at same time"; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + + opts.addOption("l", LIST_LABELS_CMD, false, + "List cluster node-label collection"); + opts.addOption("n", NODE_ID_CMD, true, "Works with -" + LIST_MAPPINGS_CMD + + ", followed by nodeId (host:port) or node (host) it will return node" + + " to label mappings when this specified. " + + "Without this option, all node" + + " to labels mappings will be returned"); + opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); + opts.addOption("m", LIST_MAPPINGS_CMD, false, + "List node-to-label mappings " + "of the cluster, by default it will " + + "return all node-to-labels mappings of the cluster."); + opts.addOption("d", DIRECTLY_ACCESS_NODE_LABEL_STORE, false, + "Directly access node label store, " + + "with this option, all node label related operations" + + " will NOT connect RM. Instead, they will" + + " access/modify stored node labels directly." + + " By default, it is false (access via RM)." + + " AND PLEASE NOTE: if you configured" + + YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR + + " to a local directory" + + " (instead of NFS or HDFS), this option will only work" + + " when the command run on the machine where RM is running."); + + int exitCode = -1; + CommandLine parsedCli = null; + try { + parsedCli = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(opts); + return exitCode; + } + + validateOptionsAndThrowExceptionIfInvalid(parsedCli); + + if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) { + accessLocal = true; + } + + if (parsedCli.hasOption(LIST_MAPPINGS_CMD)) { + String nodeIdArg = null; + if (parsedCli.hasOption(NODE_ID_CMD)) { + // print node to labels mapping + nodeIdArg = parsedCli.getOptionValue(NODE_ID_CMD); + } + printNodeToLabelsMapping(nodeIdArg); + return 0; + } else if (parsedCli.hasOption(LIST_LABELS_CMD)) { + printClusterNodeLabels(); + } else if (parsedCli.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(opts); + } + return 0; + } + + private List sortStrSet(Set labels) { + List list = new ArrayList(); + list.addAll(labels); + Collections.sort(list); + return list; + } + + private NodeId buildNodeIdFromStr(String str) { + String host = str; + int port = 0; + if (str.contains(":")) { + port = Integer.valueOf(StringUtils.substring(str, str.indexOf(':') + 1)); + host = StringUtils.substring(str, 0, str.indexOf(':')); + } + return NodeId.newInstance(host, port); + } + + private void printSingleNodeToLabelsMapping(PrintWriter pw, NodeId nodeId, + Set nodeLabels) { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("Node:%s -> ", + (nodeId.getPort() == 0 ? nodeId.getHost() : nodeId.toString()))); + + if (null != nodeLabels) { + sb.append(StringUtils.join(sortStrSet(nodeLabels), ",")); + } + pw.println(sb.toString()); + } + + void printNodeToLabelsMapping(String nodeIdStr) throws YarnException, + IOException { + Map> nodeIdToLabels = null; + + if (accessLocal) { + nodeIdToLabels = getNodeLabelManagerInstance(getConf()).getNodeLabels(); + } else { + nodeIdToLabels = client.getNodeToLabels(); + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + + List nodeIds = new ArrayList(); + + if (nodeIdStr != null) { + // User specified node id + for (String seg : nodeIdStr.split(",")) { + seg = seg.trim(); + if (seg.isEmpty()) { + continue; + } + NodeId nodeId = buildNodeIdFromStr(seg); + nodeIds.add(nodeId); + } + } else { + // User doesn't specified, will include all nodes + nodeIds.addAll(nodeIdToLabels.keySet()); + } + + // Sort nodeId + Collections.sort(nodeIds); + for (NodeId nodeId : nodeIds) { + printSingleNodeToLabelsMapping(pw, nodeId, nodeIdToLabels.get(nodeId)); + } + + pw.close(); + sysout.println(baos.toString("UTF-8")); + } + + void printClusterNodeLabels() throws YarnException, IOException { + Set nodeLabels = null; + if (accessLocal) { + nodeLabels = + getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + } else { + nodeLabels = client.getClusterNodeLabels(); + } + sysout.println(String.format("%s", + StringUtils.join(sortStrSet(nodeLabels).iterator(), ","))); + } + + // Make it protected to make unit test can change it. + protected static synchronized CommonNodeLabelsManager + getNodeLabelManagerInstance(Configuration conf) { + if (localNodeLabelsManager == null) { + localNodeLabelsManager = new CommonNodeLabelsManager(); + localNodeLabelsManager.init(conf); + localNodeLabelsManager.start(); + } + return localNodeLabelsManager; + } + + @VisibleForTesting + void printUsage(Options opts) throws UnsupportedEncodingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null, + opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, + null); + pw.close(); + sysout.println(baos.toString("UTF-8")); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeLabelsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeLabelsCLI.java new file mode 100644 index 0000000..999c07f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeLabelsCLI.java @@ -0,0 +1,300 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class TestNodeLabelsCLI { + ByteArrayOutputStream sysOutStream; + private PrintStream sysOut; + ByteArrayOutputStream sysErrStream; + private PrintStream sysErr; + + @Before + public void setup() { + sysOutStream = new ByteArrayOutputStream(); + sysOut = spy(new PrintStream(sysOutStream)); + sysErrStream = new ByteArrayOutputStream(); + sysErr = spy(new PrintStream(sysErrStream)); + System.setOut(sysOut); + } + + @Test + public void testGetClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("label1", "label2")); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-listLabels" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("remote1", "remote2")); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + NodeLabelsCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(NodeLabelsCLI.localNodeLabelsManager.getClusterNodeLabels()) + .thenReturn(ImmutableSet.of("local1", "local2")); + + int rc = + cli.run(new String[] { "node-labels", "-listLabels", + "-d" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + // it should return local* instead of remote* + pw.print("local1,local2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetEmptyClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn(new HashSet()); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-listLabels" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print(""); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetAllNodeToLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn(map); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-listMappings" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node:node1 -> label1,label2"); + pw.println("Node:node1:123 -> label5,label6"); + pw.println("Node:node2 -> label3,label4"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificHost() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn(map); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = cli.run(new String[] { "node-labels", "-listMappings", "-nodeId", "node1" } ); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node:node1 -> label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificNM() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn(map); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { "node-labels", "-listMappings", "-nodeId", "node1:123" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node:node1:123 -> label5,label6"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithSpecificNotExistedNode() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("node1", 0), ImmutableSet.of("label1", "label2")); + map.put(NodeId.newInstance("node2", 0), ImmutableSet.of("label3", "label4")); + map.put(NodeId.newInstance("node1", 123), + ImmutableSet.of("label5", "label6")); + + when(client.getNodeToLabels()).thenReturn(map); + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "node-labels", "-listMappings", "-nodeId", "node3" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node:node3 -> "); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetNodeToLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + Map> map = new HashMap>(); + map.put(NodeId.newInstance("remote1", 0), + ImmutableSet.of("label1", "label2")); + when(client.getNodeToLabels()).thenReturn(map); + map = new HashMap>(); + map.put(NodeId.newInstance("local1", 0), + ImmutableSet.of("label1", "label2")); + + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + NodeLabelsCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(NodeLabelsCLI.localNodeLabelsManager.getNodeLabels()).thenReturn(map); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "node-labels", "-listMappings", + "-directlyAccessNodeLabelStore" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node:local1 -> label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testHelp() throws Exception { + NodeLabelsCLI cli = new NodeLabelsCLI(); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "node-labels", "-help" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("usage: yarn node-labels"); + pw.println(" -d,--directlyAccessNodeLabelStore Directly access node label store,"); + pw.println(" with this option, all node label"); + pw.println(" related operations will NOT connect"); + pw.println(" RM. Instead, they will access/modify"); + pw.println(" stored node labels directly. By"); + pw.println(" default, it is false (access via RM)."); + pw.println(" AND PLEASE NOTE: if you"); + pw.println(" configuredyarn.node-labels.fs-store.r"); + pw.println(" oot-dir to a local directory (instead"); + pw.println(" of NFS or HDFS), this option will"); + pw.println(" only work when the command run on the"); + pw.println(" machine where RM is running."); + pw.println(" -h,--help Displays help for all commands."); + pw.println(" -l,--listLabels List cluster node-label collection"); + pw.println(" -m,--listMappings List node-to-label mappings of the"); + pw.println(" cluster, by default it will return"); + pw.println(" all node-to-labels mappings of the"); + pw.println(" cluster."); + pw.println(" -n,--nodeId Works with -listMappings, followed by"); + pw.println(" nodeId (host:port) or node (host) it"); + pw.println(" will return node to label mappings"); + pw.println(" when this specified. Without this"); + pw.println(" option, all node to labels mappings"); + pw.println(" will be returned"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } +} \ No newline at end of file