From 9745fc966d568e30c04a07815857863c2db8f489 Mon Sep 17 00:00:00 2001 From: Rong Tang Date: Wed, 12 Dec 2018 14:00:11 -0800 Subject: [PATCH] Add Yarn bind host for web_proxy --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 6 ++++- .../src/main/resources/yarn-default.xml | 9 +++++++ .../hadoop/yarn/server/webproxy/WebAppProxy.java | 14 +++++++++-- .../yarn/server/webproxy/WebAppProxyServer.java | 10 +++++--- .../server/webproxy/TestWebAppProxyServer.java | 29 ++++++++++++++++++---- 5 files changed, 56 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 28b54ec907..b9e09c10d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2057,13 +2057,17 @@ public static boolean isAclEnabled(Configuration conf) { /** Keytab for Proxy.*/ public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; - + /** The address for the web proxy.*/ public static final String PROXY_ADDRESS = PROXY_PREFIX + "address"; public static final int DEFAULT_PROXY_PORT = 9099; public static final String DEFAULT_PROXY_ADDRESS = "0.0.0.0:" + DEFAULT_PROXY_PORT; + + /** Binding address for the web proxy. */ + public static final String PROXY_BIND_HOST = + PROXY_PREFIX + "bind-host"; /** * YARN Service Level Authorization diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 899d5c0661..ebb599d276 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2126,6 +2126,15 @@ + + The actual address the web proxy will bind to. If this optional + address is set, it overrides only the hostname portion of yarn.web-proxy.address. + This is useful for making the web proxy server listen on all interfaces by setting + it to 0.0.0.0 + yarn.web-proxy.bind-host + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index 71679ccca9..29612c76a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -73,16 +73,26 @@ protected void serviceInit(Configuration conf) throws Exception { fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); if(bindAddress == null || bindAddress.isEmpty()) { - throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + + throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + " is not set so the proxy will not run."); } - LOG.info("Instantiating Proxy at " + bindAddress); + String[] parts = StringUtils.split(bindAddress, ':'); port = 0; if (parts.length == 2) { bindAddress = parts[0]; port = Integer.parseInt(parts[1]); } + + String bindHost = conf.getTrimmed(YarnConfiguration.PROXY_BIND_HOST, null); + if (bindHost != null) { + LOG.debug("{} is set, will be used to run proxy.", + YarnConfiguration.PROXY_BIND_HOST); + bindAddress = bindHost; + } + + LOG.info("Instantiating Proxy at {}:{}", bindAddress, port); + acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); super.serviceInit(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java index 02fcf30393..84b04a504c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java @@ -38,7 +38,7 @@ /** * ProxyServer will sit in between the end user and AppMaster - * web interfaces. + * web interfaces. */ public class WebAppProxyServer extends CompositeService { @@ -118,9 +118,11 @@ private String getHostName(Configuration config) { * @return InetSocketAddress */ public static InetSocketAddress getBindAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS, - YarnConfiguration.DEFAULT_PROXY_ADDRESS, - YarnConfiguration.DEFAULT_PROXY_PORT); + return conf.getSocketAddr( + YarnConfiguration.PROXY_BIND_HOST, + YarnConfiguration.PROXY_ADDRESS, + YarnConfiguration.DEFAULT_PROXY_ADDRESS, + YarnConfiguration.DEFAULT_PROXY_PORT); } public static void main(String[] args) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java index 93479a9b42..847419c3fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java @@ -23,7 +23,6 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -33,14 +32,15 @@ public class TestWebAppProxyServer { private WebAppProxyServer webAppProxy = null; - private final String proxyAddress = "0.0.0.0:8888"; + private final String port = "8888"; + private final String proxyAddress = "localhost:" + port; + private YarnConfiguration conf = null; @Before public void setUp() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); + conf = new YarnConfiguration(); conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress); webAppProxy = new WebAppProxyServer(); - webAppProxy.init(conf); } @After @@ -50,16 +50,35 @@ public void tearDown() throws Exception { @Test public void testStart() { + webAppProxy.init(conf); + assertEquals(STATE.INITED, webAppProxy.getServiceState()); + webAppProxy.start(); + for (Service service : webAppProxy.getServices()) { + if (service instanceof WebAppProxy) { + assertEquals(proxyAddress, ((WebAppProxy) service).getBindAddress()); + } + } + assertEquals(STATE.STARTED, webAppProxy.getServiceState()); + } + + @Test + public void testStartWithBindHost() { + String bindHost = "0.0.0.0"; + conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost); + webAppProxy.init(conf); + assertEquals(STATE.INITED, webAppProxy.getServiceState()); webAppProxy.start(); for (Service service : webAppProxy.getServices()) { if (service instanceof WebAppProxy) { - assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress); + assertEquals(bindHost + ":" + port, + ((WebAppProxy) service).getBindAddress()); } } assertEquals(STATE.STARTED, webAppProxy.getServiceState()); } + @Test public void testBindAddress() { YarnConfiguration conf = new YarnConfiguration(); -- 2.11.1.windows.1