diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index bfcb190..c976c71 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -28,6 +28,7 @@ function hadoop_usage echo " jar run a jar file" echo " logs dump container logs" echo " node prints node report(s)" + echo " cluster prints cluster information" echo " nodemanager run a nodemanager on each slave" echo " proxyserver run the web app proxy server" echo " resourcemanager run the ResourceManager" @@ -109,6 +110,11 @@ case "${COMMAND}" in hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS" YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" ;; + cluster) + CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI + hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS" + YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}" + ;; nodemanager) daemon="true" CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 0cf0b40..4a2f167 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -196,6 +196,11 @@ goto :eof set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% goto :eof +:cluster + set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager @@ -298,6 +303,7 @@ goto :eof @echo applicationattempt prints applicationattempt(s) report @echo container prints container(s) report @echo node prints node report(s) + @echo cluster prints cluster information @echo logs dump container logs @echo classpath prints the class path needed to get the @echo Hadoop jar and the required libraries diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java new file mode 100644 index 0000000..d645db2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Cluster CLI used to get over all information of the cluster + */ +public class ClusterCLI extends YarnCLI { + private static final String TITLE = "yarn cluster"; + public static final String LIST_LABELS_CMD = "listNodeLabels"; + public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = + "directlyAccessNodeLabelStore"; + public static final String CMD = "cluster"; + private boolean accessLocal = false; + static CommonNodeLabelsManager localNodeLabelsManager = null; + + public static void main(String[] args) throws Exception { + ClusterCLI cli = new ClusterCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, args); + cli.stop(); + System.exit(res); + } + + @Override + public int run(String[] args) throws Exception { + Options opts = new Options(); + + opts.addOption(LIST_LABELS_CMD, false, + "List cluster node-label collection"); + opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); + opts.addOption("d", DIRECTLY_ACCESS_NODE_LABEL_STORE, false, + "Directly access node label store, " + + "with this option, all node label related operations" + + " will NOT connect RM. Instead, they will" + + " access/modify stored node labels directly." + + " By default, it is false (access via RM)." + + " AND PLEASE NOTE: if you configured" + + YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR + + " to a local directory" + + " (instead of NFS or HDFS), this option will only work" + + " when the command run on the machine where RM is running."); + + int exitCode = -1; + CommandLine parsedCli = null; + try { + parsedCli = new GnuParser().parse(opts, args); + } catch (MissingArgumentException ex) { + sysout.println("Missing argument for options"); + printUsage(opts); + return exitCode; + } + + if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) { + accessLocal = true; + } + + if (parsedCli.hasOption(LIST_LABELS_CMD)) { + printClusterNodeLabels(); + } else if (parsedCli.hasOption(HELP_CMD)) { + printUsage(opts); + return 0; + } else { + syserr.println("Invalid Command Usage : "); + printUsage(opts); + } + return 0; + } + + private List sortStrSet(Set labels) { + List list = new ArrayList(); + list.addAll(labels); + Collections.sort(list); + return list; + } + + void printClusterNodeLabels() throws YarnException, IOException { + Set nodeLabels = null; + if (accessLocal) { + nodeLabels = + getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + } else { + nodeLabels = client.getClusterNodeLabels(); + } + sysout.println(String.format("Node Labels: %s", + StringUtils.join(sortStrSet(nodeLabels).iterator(), ","))); + } + + @VisibleForTesting + static synchronized CommonNodeLabelsManager + getNodeLabelManagerInstance(Configuration conf) { + if (localNodeLabelsManager == null) { + localNodeLabelsManager = new CommonNodeLabelsManager(); + localNodeLabelsManager.init(conf); + localNodeLabelsManager.start(); + } + return localNodeLabelsManager; + } + + @VisibleForTesting + void printUsage(Options opts) throws UnsupportedEncodingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null, + opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, + null); + pw.close(); + sysout.println(baos.toString("UTF-8")); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java new file mode 100644 index 0000000..733697d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.HashSet; + +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class TestClusterCLI { + ByteArrayOutputStream sysOutStream; + private PrintStream sysOut; + ByteArrayOutputStream sysErrStream; + private PrintStream sysErr; + + @Before + public void setup() { + sysOutStream = new ByteArrayOutputStream(); + sysOut = spy(new PrintStream(sysOutStream)); + sysErrStream = new ByteArrayOutputStream(); + sysErr = spy(new PrintStream(sysErrStream)); + System.setOut(sysOut); + } + + @Test + public void testGetClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("label1", "label2")); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node Labels: label1,label2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn( + ImmutableSet.of("remote1", "remote2")); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); + when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels()) + .thenReturn(ImmutableSet.of("local1", "local2")); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, + "-" + ClusterCLI.LIST_LABELS_CMD, + "-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + // it should return local* instead of remote* + pw.print("Node Labels: local1,local2"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testGetEmptyClusterNodeLabels() throws Exception { + YarnClient client = mock(YarnClient.class); + when(client.getClusterNodeLabels()).thenReturn(new HashSet()); + ClusterCLI cli = new ClusterCLI(); + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + int rc = + cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.print("Node Labels: "); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } + + @Test + public void testHelp() throws Exception { + ClusterCLI cli = new ClusterCLI(); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + + // node 3 isn't existed .. + int rc = + cli.run(new String[] { "cluster", "-help" }); + assertEquals(0, rc); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("usage: yarn cluster"); + pw.println(" -d,--directlyAccessNodeLabelStore Directly access node label store,"); + pw.println(" with this option, all node label"); + pw.println(" related operations will NOT connect"); + pw.println(" RM. Instead, they will access/modify"); + pw.println(" stored node labels directly. By"); + pw.println(" default, it is false (access via RM)."); + pw.println(" AND PLEASE NOTE: if you"); + pw.println(" configuredyarn.node-labels.fs-store.r"); + pw.println(" oot-dir to a local directory (instead"); + pw.println(" of NFS or HDFS), this option will"); + pw.println(" only work when the command run on the"); + pw.println(" machine where RM is running."); + pw.println(" -h,--help Displays help for all commands."); + pw.println(" -listNodeLabels List cluster node-label collection"); + pw.close(); + verify(sysOut).println(baos.toString("UTF-8")); + } +} \ No newline at end of file