From b17db0104b2a25b2b0b62148e2e062c678bfa18e Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 26 Aug 2019 00:38:32 +0530 Subject: [PATCH] YARN-9781. SchedConfCli to get current stored scheduler configuration. --- .../hadoop/yarn/client/cli/SchedConfCLI.java | 78 +++++++++++++++++++++- .../hadoop/yarn/webapp/util/WebAppUtils.java | 28 ++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index be54553..0303a32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -32,13 +32,24 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -56,6 +67,7 @@ private static final String REMOVE_QUEUES_OPTION = "removeQueues"; private static final String UPDATE_QUEUES_OPTION = "updateQueues"; private static final String GLOBAL_OPTIONS = "globalUpdates"; + private static final String GET_SCHEDULER_CONF = "getConf"; private static final String HELP_CMD = "help"; private static final String CONF_ERR_MSG = "Specify configuration key " + @@ -83,6 +95,8 @@ public int run(String[] args) throws Exception { "Update queue configurations"); opts.addOption("global", GLOBAL_OPTIONS, true, "Update global scheduler configurations"); + opts.addOption("getconf", GET_SCHEDULER_CONF, false, + "Get current scheduler configurations"); opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); int exitCode = -1; @@ -101,6 +115,7 @@ public int run(String[] args) throws Exception { } boolean hasOption = false; + boolean getConf = false; SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); try { if (parsedCli.hasOption(ADD_QUEUES_OPTION)) { @@ -121,6 +136,11 @@ public int run(String[] args) throws Exception { hasOption = true; globalUpdates(parsedCli.getOptionValue(GLOBAL_OPTIONS), updateInfo); } + if (parsedCli.hasOption(GET_SCHEDULER_CONF)) { + hasOption = true; + getConf = true; + } + } catch (IllegalArgumentException e) { System.err.println(e.getMessage()); return -1; @@ -133,8 +153,59 @@ public int run(String[] args) throws Exception { } Configuration conf = getConf(); - return WebAppUtils.execOnActiveRM(conf, - this::updateSchedulerConfOnRMNode, updateInfo); + if (getConf) { + return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf); + } else { + return WebAppUtils.execOnActiveRM(conf, + this::updateSchedulerConfOnRMNode, updateInfo); + } + } + + public static void prettyFormat(String input, int indent) + throws Exception { + Source xmlInput = new StreamSource(new StringReader(input)); + StringWriter sw = new StringWriter(); + StreamResult xmlOutput = new StreamResult(sw); + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformerFactory.setAttribute("indent-number", indent); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.transform(xmlInput, xmlOutput); + System.out.println(xmlOutput.getWriter().toString()); + } + + private int getSchedulerConf(String webAppAddress) throws Exception { + Client webServiceClient = Client.create(); + ClientResponse response = null; + try { + Builder builder = webServiceClient.resource(webAppAddress) + .path("ws").path("v1").path("cluster") + .path("scheduler-conf").accept(MediaType.APPLICATION_XML); + response = builder.get(ClientResponse.class); + if (response != null) { + if (response.getStatus() == Status.OK.getStatusCode()) { + ConfInfo schedulerConf = response.getEntity(ConfInfo.class); + JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + StringWriter sw = new StringWriter(); + jaxbMarshaller.marshal(schedulerConf, sw); + prettyFormat(sw.toString(), 2); + return 0; + } else { + System.err.println("Failed to get scheduler configuration: " + + response.getEntity(String.class)); + } + } else { + System.err.println("Failed to get scheduler configuration: " + + "null response"); + } + return -1; + } finally { + if (response != null) { + response.close(); + } + webServiceClient.destroy(); + } } private int updateSchedulerConfOnRMNode(String webAppAddress, @@ -214,6 +285,7 @@ void globalUpdates(String args, SchedConfUpdateInfo updateInfo) { updateInfo.setGlobalParams(globalUpdates); } + private QueueConfigInfo getQueueConfigInfo(String arg) { String[] args = arg.split(":"); String queuePath = args[0]; @@ -264,6 +336,8 @@ private void printUsage() { + "maximum-capacity=75\"\n" + "Example (global scheduler update): yarn schedulerconf " + "-global yarn.scheduler.capacity.maximum-applications=10000\n" + + "Example (get scheduler configuration): yarn schedulerconf " + + "-getconf\n" + "Note: This is an alpha feature, the syntax/options are subject to " + "change, please run at your own risk."); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 09daf42..625c6de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -123,6 +123,34 @@ public static void setNMWebAppHostNameAndPort(Configuration conf, R apply(T t, U u) throws Exception; } + /** A Function which throws on Exception. */ + @FunctionalInterface + public interface ThrowingFunction { + R apply(T t) throws Exception; + } + + public static R execOnActiveRM(Configuration conf, + ThrowingFunction func) throws Exception { + String rm1Address = getRMWebAppURLWithScheme(conf, 0); + try { + return func.apply(rm1Address); + } catch (Exception e) { + if (HAUtil.isHAEnabled(conf)) { + int rms = HAUtil.getRMHAIds(conf).size(); + for (int i=1; i