From f4838732d24abbbdd4c57a4c74a523d9c0e300a9 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Tue, 27 Aug 2019 14:01:10 +0530 Subject: [PATCH] YARN-9781. SchedConfCli to get current stored scheduler configuration. --- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 11 ++ .../hadoop/yarn/client/cli/SchedConfCLI.java | 84 ++++++++++- .../hadoop/yarn/client/cli/TestSchedConfCLI.java | 167 ++++++++++++++++++++- .../apache/hadoop/yarn/webapp/dao/ConfInfo.java | 78 ++++++++++ .../hadoop/yarn/webapp/util/WebAppUtils.java | 28 ++++ .../resourcemanager/webapp/RMWebServices.java | 2 +- .../resourcemanager/webapp/dao/ConfInfo.java | 72 --------- 7 files changed, 366 insertions(+), 76 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 81ff752..30275b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -65,6 +65,17 @@ mockito-core test + + com.sun.jersey.jersey-test-framework + jersey-test-framework-core + test + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + test + + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java index be54553..bbb5d68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.WebResource.Builder; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -32,13 +33,24 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.dao.ConfInfo; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -56,6 +68,7 @@ private static final String REMOVE_QUEUES_OPTION = "removeQueues"; private static final String UPDATE_QUEUES_OPTION = "updateQueues"; private static final String GLOBAL_OPTIONS = "globalUpdates"; + private static final String GET_SCHEDULER_CONF = "getConf"; private static final String HELP_CMD = "help"; private static final String CONF_ERR_MSG = "Specify configuration key " + @@ -83,6 +96,8 @@ public int run(String[] args) throws Exception { "Update queue configurations"); opts.addOption("global", GLOBAL_OPTIONS, true, "Update global scheduler configurations"); + opts.addOption("getconf", GET_SCHEDULER_CONF, false, + "Get current scheduler configurations"); opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); int exitCode = -1; @@ -101,6 +116,7 @@ public int run(String[] args) throws Exception { } boolean hasOption = false; + boolean getConf = false; SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); try { if (parsedCli.hasOption(ADD_QUEUES_OPTION)) { @@ -121,6 +137,11 @@ public int run(String[] args) throws Exception { hasOption = true; globalUpdates(parsedCli.getOptionValue(GLOBAL_OPTIONS), updateInfo); } + if (parsedCli.hasOption(GET_SCHEDULER_CONF)) { + hasOption = true; + getConf = true; + } + } catch (IllegalArgumentException e) { System.err.println(e.getMessage()); return -1; @@ -133,8 +154,64 @@ public int run(String[] args) throws Exception { } Configuration conf = getConf(); - return WebAppUtils.execOnActiveRM(conf, - this::updateSchedulerConfOnRMNode, updateInfo); + if (getConf) { + return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf, null); + } else { + return WebAppUtils.execOnActiveRM(conf, + this::updateSchedulerConfOnRMNode, updateInfo); + } + } + + public static void prettyFormat(String input, int indent) + throws Exception { + Source xmlInput = new StreamSource(new StringReader(input)); + StringWriter sw = new StringWriter(); + StreamResult xmlOutput = new StreamResult(sw); + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + transformerFactory.setAttribute("indent-number", indent); + Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.transform(xmlInput, xmlOutput); + System.out.println(xmlOutput.getWriter().toString()); + } + + + @VisibleForTesting + int getSchedulerConf(String webAppAddress, WebResource resource) + throws Exception { + Client webServiceClient = Client.create(); + ClientResponse response = null; + resource = (resource != null) ? resource : + webServiceClient.resource(webAppAddress); + try { + Builder builder = resource + .path("ws").path("v1").path("cluster") + .path("scheduler-conf").accept(MediaType.APPLICATION_XML); + response = builder.get(ClientResponse.class); + if (response != null) { + if (response.getStatus() == Status.OK.getStatusCode()) { + ConfInfo schedulerConf = response.getEntity(ConfInfo.class); + JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + StringWriter sw = new StringWriter(); + jaxbMarshaller.marshal(schedulerConf, sw); + prettyFormat(sw.toString(), 2); + return 0; + } else { + System.err.println("Failed to get scheduler configuration: " + + response.getEntity(String.class)); + } + } else { + System.err.println("Failed to get scheduler configuration: " + + "null response"); + } + return -1; + } finally { + if (response != null) { + response.close(); + } + webServiceClient.destroy(); + } } private int updateSchedulerConfOnRMNode(String webAppAddress, @@ -214,6 +291,7 @@ void globalUpdates(String args, SchedConfUpdateInfo updateInfo) { updateInfo.setGlobalParams(globalUpdates); } + private QueueConfigInfo getQueueConfigInfo(String arg) { String[] args = arg.split(":"); String queuePath = args[0]; @@ -264,6 +342,8 @@ private void printUsage() { + "maximum-capacity=75\"\n" + "Example (global scheduler update): yarn schedulerconf " + "-global yarn.scheduler.capacity.maximum-applications=10000\n" + + "Example (get scheduler configuration): yarn schedulerconf " + + "-getconf\n" + "Note: This is an alpha feature, the syntax/options are subject to " + "change, please run at your own risk."); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 5364e83..1fd6c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -18,16 +18,49 @@ package org.apache.hadoop.yarn.client.cli; +import com.google.inject.Guice; +import com.google.inject.Singleton; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.GuiceServletConfig; +import org.apache.hadoop.yarn.webapp.JerseyTestBase; + import org.junit.Before; import org.junit.Test; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import java.io.PrintStream; +import java.security.Principal; import java.util.List; import java.util.Map; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletResponse; +import javax.servlet.ServletRequest; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import javax.servlet.http.HttpServletResponse; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -35,7 +68,7 @@ /** * Class for testing {@link SchedConfCLI}. */ -public class TestSchedConfCLI { +public class TestSchedConfCLI extends JerseyTestBase { private ByteArrayOutputStream sysOutStream; private PrintStream sysOut; @@ -45,6 +78,23 @@ private SchedConfCLI cli; + private static MockRM rm; + private static String userName; + private static CapacitySchedulerConfiguration csConf; + + private static final File CONF_FILE = new File(new File("target", + "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE); + private static final File OLD_CONF_FILE = new File(new File("target", + "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp"); + + public TestSchedConfCLI() { + super(new WebAppDescriptor.Builder( + "org.apache.hadoop.yarn.server.resourcemanager.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); + } + @Before public void setUp() { sysOutStream = new ByteArrayOutputStream(); @@ -58,6 +108,121 @@ public void setUp() { cli = new SchedConfCLI(); } + private static class WebServletModule extends ServletModule { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + + try { + userName = UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException ioe) { + throw new RuntimeException("Unable to get current user name " + + ioe.getMessage(), ioe); + } + + csConf = new CapacitySchedulerConfiguration(new Configuration(false), + false); + setupQueueConfiguration(csConf); + + + try { + if (CONF_FILE.exists()) { + if (!CONF_FILE.renameTo(OLD_CONF_FILE)) { + throw new RuntimeException("Failed to rename conf file"); + } + } + FileOutputStream out = new FileOutputStream(CONF_FILE); + csConf.writeXml(out); + out.close(); + } catch (IOException e) { + throw new RuntimeException("Failed to write XML file", e); + } + + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + serve("/*").with(GuiceContainer.class); + filter("/*").through(TestRMCustomAuthFilter.class); + } + } + + /** + * Custom filter which sets the Remote User for testing purpose. + */ + @Singleton + public static class TestRMCustomAuthFilter extends AuthenticationFilter { + @Override + public void init(FilterConfig filterConfig) { + + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain filterChain) throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest)request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + httpRequest = new HttpServletRequestWrapper(httpRequest) { + public String getAuthType() { + return null; + } + + public String getRemoteUser() { + return userName; + } + + public Principal getUserPrincipal() { + return new Principal() { + @Override + public String getName() { + return userName; + } + }; + } + }; + doFilter(filterChain, httpRequest, httpResponse); + } + } + + private static void setupQueueConfiguration( + CapacitySchedulerConfiguration config) { + config.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{"testqueue"}); + String a = CapacitySchedulerConfiguration.ROOT + ".testqueue"; + config.setCapacity(a, 100f); + config.setMaximumCapacity(a, 100f); + } + + @Test(timeout = 10000) + public void testGetSchedulerConf() throws Exception { + try { + super.setUp(); + GuiceServletConfig.setInjector( + Guice.createInjector(new WebServletModule())); + int exitCode = cli.getSchedulerConf("", resource()); + assertEquals(0, exitCode); + assertTrue("Failed to get scheduler configuration", + sysOutStream.toString().contains("testqueue")); + } finally { + if (rm != null) { + rm.stop(); + } + CONF_FILE.delete(); + if (OLD_CONF_FILE.exists()) { + if (!OLD_CONF_FILE.renameTo(CONF_FILE)) { + throw new RuntimeException("Failed to re-copy old" + + " configuration file"); + } + } + super.tearDown(); + } + } + @Test(timeout = 10000) public void testInvalidConf() throws Exception { // conf pair with no key should be invalid diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java new file mode 100644 index 0000000..1971efa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.webapp.dao; + +import org.apache.hadoop.conf.Configuration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +/** + * This class stores the Scheduler Configuration. + */ +@XmlRootElement(name = "configuration") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfInfo { + + private ArrayList property = new ArrayList<>(); + + public ConfInfo() { + } // JAXB needs this + + public ConfInfo(Configuration conf) { + conf.forEach(entry -> + add(new ConfItem(entry.getKey(), entry.getValue()))); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + /** + * This class stores the Configuration Property. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 09daf42..625c6de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -123,6 +123,34 @@ public static void setNMWebAppHostNameAndPort(Configuration conf, R apply(T t, U u) throws Exception; } + /** A Function which throws on Exception. */ + @FunctionalInterface + public interface ThrowingFunction { + R apply(T t) throws Exception; + } + + public static R execOnActiveRM(Configuration conf, + ThrowingFunction func) throws Exception { + String rm1Address = getRMWebAppURLWithScheme(conf, 0); + try { + return func.apply(rm1Address); + } catch (Exception e) { + if (HAUtil.isHAEnabled(conf)) { + int rms = HAUtil.getRMHAIds(conf).size(); + for (int i=1; i property = new ArrayList<>(); - - public ConfInfo() { - } // JAXB needs this - - public ConfInfo(Configuration conf) { - conf.forEach(entry -> - add(new ConfItem(entry.getKey(), entry.getValue()))); - } - - public void add(ConfItem confItem) { - property.add(confItem); - } - - public ArrayList getItems() { - return property; - } - - @XmlAccessorType(XmlAccessType.FIELD) - public static class ConfItem { - - private String name; - private String value; - - public ConfItem() { - // JAXB needs this - } - - public ConfItem(String name, String value){ - this.name = name; - this.value = value; - } - - public String getKey() { - return name; - } - - public String getValue() { - return value; - } - } -} -- 2.7.4 (Apple Git-66)