From 8a61a5957c31c6b33942766f9d20ea49c8ebcf94 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 18 Oct 2019 18:26:41 +0530 Subject: [PATCH] YARN-9781. SchedConfCli to get current stored scheduler configuration. --- .../hadoop/yarn/client/cli/SchedConfCLI.java | 95 +++++++++++++++++++++- .../hadoop/yarn/client/cli/TestSchedConfCLI.java | 25 ++++++ .../apache/hadoop/yarn/webapp/dao/ConfInfo.java | 78 ++++++++++++++++++ .../resourcemanager/webapp/RMWebServices.java | 2 +- .../resourcemanager/webapp/dao/ConfInfo.java | 72 ---------------- 5 files changed, 198 insertions(+), 74 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index 146db9e..9370709 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -38,13 +38,24 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import java.io.StringReader; +import java.io.StringWriter; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; @@ -65,6 +76,7 @@ private static final String REMOVE_QUEUES_OPTION = "removeQueues"; private static final String UPDATE_QUEUES_OPTION = "updateQueues"; private static final String GLOBAL_OPTIONS = "globalUpdates"; + private static final String GET_SCHEDULER_CONF = "getConf"; private static final String FORMAT_CONF = "formatConfig"; private static final String HELP_CMD = "help"; @@ -93,6 +105,8 @@ public int run(String[] args) throws Exception { "Update queue configurations"); opts.addOption("global", GLOBAL_OPTIONS, true, "Update global scheduler configurations"); + opts.addOption("getconf", GET_SCHEDULER_CONF, false, + "Get current scheduler configurations"); opts.addOption("format", FORMAT_CONF, false, "Format Scheduler Configuration and reload from" + " capacity-scheduler.xml"); @@ -115,6 +129,7 @@ public int run(String[] args) throws Exception { boolean hasOption = false; boolean format = false; + boolean getConf = false; SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); try { if (parsedCli.hasOption(ADD_QUEUES_OPTION)) { @@ -139,6 +154,10 @@ public int run(String[] args) throws Exception { hasOption = true; format = true; } + if (parsedCli.hasOption(GET_SCHEDULER_CONF)) { + hasOption = true; + getConf = true; + } } catch (IllegalArgumentException e) { System.err.println(e.getMessage()); @@ -154,12 +173,83 @@ public int run(String[] args) throws Exception { Configuration conf = getConf(); if (format) { return WebAppUtils.execOnActiveRM(conf, this::formatSchedulerConf, null); + } else if (getConf) { + return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf, null); } else { return WebAppUtils.execOnActiveRM(conf, this::updateSchedulerConfOnRMNode, updateInfo); } } + public static void prettyFormat(String input, int indent) + throws Exception { + Source xmlInput = new StreamSource(new StringReader(input)); + StringWriter sw = new StringWriter(); + StreamResult xmlOutput = new StreamResult(sw); + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformerFactory.setAttribute("indent-number", indent); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.transform(xmlInput, xmlOutput); + System.out.println(xmlOutput.getWriter().toString()); + } + + @VisibleForTesting + int getSchedulerConf(String webAppAddress, WebResource resource) + throws Exception { + Configuration conf = getConf(); + SSLFactory clientSslFactory = null; + if (YarnConfiguration.useHttps(conf)) { + clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + } + Client webServiceClient = createWebServiceClient(clientSslFactory); + ClientResponse response = null; + resource = (resource != null) ? resource : + webServiceClient.resource(webAppAddress); + try { + Builder builder = null; + if (UserGroupInformation.isSecurityEnabled()) { + builder = resource + .path("ws").path("v1").path("cluster") + .path("scheduler-conf").accept(MediaType.APPLICATION_XML); + } else { + builder = resource + .path("ws").path("v1").path("cluster").path("scheduler-conf") + .queryParam("user.name", UserGroupInformation.getCurrentUser() + .getShortUserName()).accept(MediaType.APPLICATION_XML); + } + response = builder.get(ClientResponse.class); + if (response != null) { + if (response.getStatus() == Status.OK.getStatusCode()) { + ConfInfo schedulerConf = response.getEntity(ConfInfo.class); + JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + StringWriter sw = new StringWriter(); + jaxbMarshaller.marshal(schedulerConf, sw); + prettyFormat(sw.toString(), 2); + return 0; + } else { + System.err.println("Failed to get scheduler configuration: " + + response.getEntity(String.class)); + } + } else { + System.err.println("Failed to get scheduler configuration: " + + "null response"); + } + return -1; + } finally { + if (response != null) { + response.close(); + } + if (webServiceClient != null) { + webServiceClient.destroy(); + } + if (clientSslFactory != null) { + clientSslFactory.destroy(); + } + } + } + @VisibleForTesting int formatSchedulerConf(String webAppAddress, WebResource resource) throws Exception { @@ -380,7 +470,8 @@ private void printUsage() { + "[-update \"queueUpdatePath1:confKey1=confVal1\"] " + "[-global globalConfKey1=globalConfVal1," + "globalConfKey2=globalConfVal2] " - + "[-format]\n" + + "[-format] " + + "[-getconf]\n" + "Example (adding queues): yarn schedulerconf -add " + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0," + "maximum-capacity=0\"\n" @@ -393,6 +484,8 @@ private void printUsage() { + "-global yarn.scheduler.capacity.maximum-applications=10000\n" + "Example (format scheduler configuration): yarn schedulerconf " + "-format\n" + + "Example (get scheduler configuration): yarn schedulerconf " + + "-getconf\n" + "Note: This is an alpha feature, the syntax/options are subject to " + "change, please run at your own risk."); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 8a6c9c2..37bc0b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -201,6 +201,31 @@ private static void setupQueueConfiguration( } @Test(timeout = 10000) + public void testGetSchedulerConf() throws Exception { + try { + super.setUp(); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); + int exitCode = cli.getSchedulerConf("", resource()); + assertEquals(0, exitCode); + assertTrue("Failed to get scheduler configuration", + sysOutStream.toString().contains("testqueue")); + } finally { + if (rm != null) { + rm.stop(); + } + CONF_FILE.delete(); + if (OLD_CONF_FILE.exists()) { + if (!OLD_CONF_FILE.renameTo(CONF_FILE)) { + throw new RuntimeException("Failed to re-copy old" + + " configuration file"); + } + } + super.tearDown(); + } + } + + @Test(timeout = 10000) public void testFormatSchedulerConf() throws Exception { try { super.setUp(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java new file mode 100644 index 0000000..1971efa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.webapp.dao; + +import org.apache.hadoop.conf.Configuration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +/** + * This class stores the Scheduler Configuration. + */ +@XmlRootElement(name = "configuration") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfInfo { + + private ArrayList property = new ArrayList<>(); + + public ConfInfo() { + } // JAXB needs this + + public ConfInfo(Configuration conf) { + conf.forEach(entry -> + add(new ConfItem(entry.getKey(), entry.getValue()))); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + /** + * This class stores the Configuration Property. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d263830..0447289 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -197,7 +197,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -210,6 +209,7 @@ import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java deleted file mode 100644 index bfc450b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; - -import org.apache.hadoop.conf.Configuration; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import java.util.ArrayList; - -@XmlRootElement(name = "configuration") -@XmlAccessorType(XmlAccessType.FIELD) -public class ConfInfo { - - protected ArrayList property = new ArrayList<>(); - - public ConfInfo() { - } // JAXB needs this - - public ConfInfo(Configuration conf) { - conf.forEach(entry -> - add(new ConfItem(entry.getKey(), entry.getValue()))); - } - - public void add(ConfItem confItem) { - property.add(confItem); - } - - public ArrayList getItems() { - return property; - } - - @XmlAccessorType(XmlAccessType.FIELD) - public static class ConfItem { - - private String name; - private String value; - - public ConfItem() { - // JAXB needs this - } - - public ConfItem(String name, String value){ - this.name = name; - this.value = value; - } - - public String getKey() { - return name; - } - - public String getValue() { - return value; - } - } -} -- 2.7.4 (Apple Git-66)