diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 074e3f0..5f39edd 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -141,7 +141,9 @@ protected void startRpcServer() {
}
server.start();
- this.address = NetUtils.getConnectAddress(server);
+ this.address = NetUtils.createSocketAddrForHost(
+ context.getNMHostname(),
+ server.getListenerAddress().getPort());
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
index 6f036c4..31e282a 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java
@@ -66,4 +66,5 @@
boolean hasSuccessfullyUnregistered();
+ String getNMHostname();
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 8c1892a..59e7249 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1018,6 +1018,11 @@ public void markSuccessfulUnregistration() {
public void resetIsLastAMRetry() {
isLastAMRetry = false;
}
+
+ @Override
+ public String getNMHostname() {
+ return nmHost;
+ }
}
@SuppressWarnings("unchecked")
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index 3c0e100..1123532 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -131,7 +131,8 @@ protected void serviceStart() throws Exception {
}
server.start();
- this.bindAddress = NetUtils.getConnectAddress(server);
+ this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
+ server.getListenerAddress().getPort());
LOG.info("Instantiated MRClientService at " + this.bindAddress);
try {
// Explicitly disabling SSL for map reduce task as we can't allow MR users
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
index 256f0b7..2c81cf0 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
@@ -63,6 +63,13 @@
public MockTaskAttemptListenerImpl(AppContext context,
JobTokenSecretManager jobTokenSecretManager,
+ RMHeartbeatHandler rmHeartbeatHandler, AMPreemptionPolicy policy) {
+
+ super(context, jobTokenSecretManager, rmHeartbeatHandler, policy);
+ }
+
+ public MockTaskAttemptListenerImpl(AppContext context,
+ JobTokenSecretManager jobTokenSecretManager,
RMHeartbeatHandler rmHeartbeatHandler,
TaskHeartbeatHandler hbHandler,
AMPreemptionPolicy policy) {
@@ -210,7 +217,7 @@ public void testGetMapCompletionEvents() throws IOException {
when(appCtx.getEventHandler()).thenReturn(ea);
CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
policy.init(appCtx);
- TaskAttemptListenerImpl listener = new TaskAttemptListenerImpl(
+ TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(
appCtx, secret, rmHeartbeatHandler, policy) {
@Override
protected void registerHeartbeatHandler(Configuration conf) {
@@ -271,7 +278,7 @@ public void testCommitWindow() throws IOException {
when(appCtx.getEventHandler()).thenReturn(ea);
CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
policy.init(appCtx);
- TaskAttemptListenerImpl listener = new TaskAttemptListenerImpl(
+ TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(
appCtx, secret, rmHeartbeatHandler, policy) {
@Override
protected void registerHeartbeatHandler(Configuration conf) {
@@ -326,7 +333,7 @@ public void testCheckpointIDTracking()
when(appCtx.getEventHandler()).thenReturn(ea);
CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
policy.init(appCtx);
- TaskAttemptListenerImpl listener = new TaskAttemptListenerImpl(
+ TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(
appCtx, secret, rmHeartbeatHandler, policy) {
@Override
protected void registerHeartbeatHandler(Configuration conf) {
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
index 511731a..dae0aa7 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
@@ -143,4 +143,9 @@ public boolean hasSuccessfullyUnregistered() {
return true;
}
+@Override
+ public String getNMHostname() {
+ // bogus - Not Required
+ return null;
+ }
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index 6fadf35..c25cf50 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -879,5 +879,10 @@ public boolean hasSuccessfullyUnregistered() {
return true;
}
+ @Override
+ public String getNMHostname() {
+ // bogus - Not Required
+ return null;
+ }
}
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 2e1a22e..9fa8a09 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -38,7 +38,9 @@
public static final int DEFAULT_MR_HISTORY_PORT = 10020;
public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:" +
DEFAULT_MR_HISTORY_PORT;
-
+ public static final String MR_HISTORY_BIND_HOST = MR_HISTORY_PREFIX
+ + "bind-host";
+
/** The address of the History server admin interface. */
public static final String JHS_ADMIN_ADDRESS = MR_HISTORY_PREFIX
+ "admin.address";
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
index 2d453f1..295ed01 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -105,11 +106,15 @@ public static String getJHSWebappURLWithScheme(Configuration conf) {
public static InetSocketAddress getJHSWebBindAddress(Configuration conf) {
if (httpPolicyInJHS == Policy.HTTPS_ONLY) {
- return conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+ return RPCUtil.getSocketAddr(conf,
+ JHAdminConfig.MR_HISTORY_BIND_HOST,
+ JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
} else {
- return conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ return RPCUtil.getSocketAddr(conf,
+ JHAdminConfig.MR_HISTORY_BIND_HOST,
+ JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 96b8105..9615876 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -83,6 +83,7 @@
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.webapp.WebApp;
@@ -118,7 +119,9 @@ protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
initializeWebApp(conf);
- InetSocketAddress address = conf.getSocketAddr(
+ InetSocketAddress address = RPCUtil.getSocketAddr(
+ conf,
+ JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.MR_HISTORY_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_PORT);
@@ -137,9 +140,11 @@ protected void serviceStart() throws Exception {
}
server.start();
- this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_ADDRESS,
- server.getListenerAddress());
- LOG.info("Instantiated MRClientService at " + this.bindAddress);
+ this.bindAddress = RPCUtil.updateConnectAddr(conf,
+ JHAdminConfig.MR_HISTORY_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
+ server.getListenerAddress());
+ LOG.info("Instantiated HistoryClientService at " + this.bindAddress);
super.serviceStart();
}
@@ -158,8 +163,9 @@ protected void initializeWebApp(Configuration conf) {
JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY)
.at(NetUtils.getHostPortString(bindAddress)).start(webApp);
+ String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
- NetUtils.getHostPortString(webApp.getListenerAddress()));
+ connectHost + ":" + webApp.getListenerAddress().getPort());
}
@Override
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
index b7823a0..194b85a 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
@@ -394,4 +394,9 @@ public boolean hasSuccessfullyUnregistered() {
return true;
}
+ @Override
+ public String getNMHostname() {
+ // bogus - Not Required
+ return null;
+ }
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 23a34a4..2a22fb4 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserMappingsProtocolService;
import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
@@ -94,7 +95,9 @@ public void serviceInit(Configuration conf) throws Exception {
WritableRpcEngine.ensureInitialized();
- clientRpcAddress = conf.getSocketAddr(JHAdminConfig.JHS_ADMIN_ADDRESS,
+ clientRpcAddress = RPCUtil.getSocketAddr(conf,
+ JHAdminConfig.MR_HISTORY_BIND_HOST,
+ JHAdminConfig.JHS_ADMIN_ADDRESS,
JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);
clientRpcServer = new RPC.Builder(conf)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 59e108a..902484b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -120,12 +120,16 @@
public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
/** The address of the applications manager interface in the RM.*/
- public static final String RM_ADDRESS =
+ public static final String RM_ADDRESS =
RM_PREFIX + "address";
public static final int DEFAULT_RM_PORT = 8032;
public static final String DEFAULT_RM_ADDRESS =
"0.0.0.0:" + DEFAULT_RM_PORT;
+ /** The actual bind address for the RM.*/
+ public static final String RM_BIND_HOST =
+ RM_PREFIX + "bind-host";
+
/** The number of threads used to handle applications manager requests.*/
public static final String RM_CLIENT_THREAD_COUNT =
RM_PREFIX + "client.thread-count";
@@ -533,7 +537,11 @@
public static final int DEFAULT_NM_PORT = 0;
public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:"
+ DEFAULT_NM_PORT;
-
+
+ /** The actual bind address or the NM.*/
+ public static final String NM_BIND_HOST =
+ NM_PREFIX + "bind-host";
+
/** who will execute(launch) the containers.*/
public static final String NM_CONTAINER_EXECUTOR =
NM_PREFIX + "container-executor.class";
@@ -1121,6 +1129,10 @@
public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:"
+ DEFAULT_TIMELINE_SERVICE_PORT;
+ /** The listening endpoint for the timeline service application.*/
+ public static final String TIMELINE_SERVICE_BIND_HOST =
+ TIMELINE_SERVICE_PREFIX + "bind-host";
+
/** The number of threads to handle client RPC API requests. */
public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
TIMELINE_SERVICE_PREFIX + "handler-thread-count";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
index ada0669..0be4834 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
@@ -21,10 +21,14 @@
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.net.InetSocketAddress;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import com.google.protobuf.ServiceException;
@@ -124,4 +128,94 @@ public static Void unwrapAndThrowException(ServiceException se)
}
}
}
+ /**
+ * Get the socket address for hostProperty as a
+ * InetSocketAddress. If hostProperty is
+ * null, addressProperty will be used. This
+ * is useful for cases where we want to differentiate between host
+ * bind address and address clients should use to establish connection.
+ *
+ * @param conf the configuration.
+ * @param hostProperty bind host property name.
+ * @param addressProperty address property name.
+ * @param defaultAddress the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ public static InetSocketAddress getSocketAddr(
+ Configuration conf,
+ String hostProperty,
+ String addressProperty,
+ String defaultAddress,
+ int defaultPort) {
+
+ InetSocketAddress bindAddr = conf.getSocketAddr(
+ addressProperty, defaultAddress, defaultPort);
+
+ final String host = conf.get(hostProperty);
+
+ if (host == null || host.isEmpty()) {
+ return bindAddr;
+ }
+
+ return NetUtils.createSocketAddr(
+ host, bindAddr.getPort(), hostProperty);
+ }
+
+ /**
+ * Set the socket address a client can use to connect for the
+ * property property as a host:port. The
+ * listening port of the server address will be used with the hostname
+ * of the client address to construct the connect address. The wildcard
+ * address is replaced with the local host's address.
+ *
+ * @param conf the configuration
+ * @param property property name for the connect address
+ * @param defaultValue default property for connect address
+ * @param serverAddr InetSocketAddress to be used as connect address
+ * @return InetSocketAddress for clients to connect
+ */
+ public static InetSocketAddress updateConnectAddr(
+ Configuration conf,
+ String property,
+ String defaultValue,
+ InetSocketAddress serverAddr) {
+
+ String connectHost = conf.getTrimmed(property, defaultValue).split(":")[0];
+ // Create connect address using client address hostname and server port.
+ return conf.updateConnectAddr(property, NetUtils.createSocketAddrForHost(
+ connectHost, serverAddr.getPort()));
+ }
+
+ /**
+ * Get the address to use for binding where bind hostname can be specified
+ * to override the hostname in the connect address. Port specified in the
+ * address will be used.
+ *
+ * @param conf the configuration
+ * @param hostProperty bind host property name
+ * @param address connect address String
+ * @param defaultPort default value for port
+ * @return String representing bind address
+ */
+ public static String getAddressAsString(
+ Configuration conf,
+ String hostProperty,
+ String address,
+ int defaultPort) {
+
+ // If the bind-host setting exists then it overrides the hostname
+ // portion of the corresponding address.
+ String host = conf.getTrimmed(hostProperty);
+ if (host != null && !host.isEmpty()) {
+ if (address.contains(":")) {
+ address = host + ":" + address.split(":")[1];
+ }
+ else {
+ address = host + ":" + defaultPort;
+ }
+ }
+
+ return address;
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 29fd8c1..b01f79b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.util.RMHAUtils;
@Private
@@ -85,6 +86,22 @@ public static String getRMWebAppURLWithoutScheme(Configuration conf) {
}
}
+ public static String getRMWebAppBindURLWithoutScheme(Configuration conf) {
+ if (YarnConfiguration.useHttps(conf)) {
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
+ }else {
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ }
+ }
+
public static List getProxyHostsAndPortsForAmFilter(
Configuration conf) {
List addrs = new ArrayList();
@@ -169,14 +186,20 @@ private static String getResolvedAddress(InetSocketAddress address) {
sb.append(":").append(address.getPort());
return sb.toString();
}
-
- public static String getNMWebAppURLWithoutScheme(Configuration conf) {
+
+ public static String getNMWebAppBindURLWithoutScheme(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
- return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS);
- } else {
- return conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.NM_BIND_HOST,
+ conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS),
+ YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_PORT);
+ }else {
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.NM_BIND_HOST,
+ conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_NM_WEBAPP_PORT);
}
}
@@ -189,7 +212,23 @@ public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
}
-
+
+ public static String getAHSWebAppBindURLWithoutScheme(Configuration conf) {
+ if (YarnConfiguration.useHttps(conf)) {
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT);
+ }else {
+ return RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT);
+ }
+ }
+
/**
* if url has scheme then it will be returned as it is else it will return
* url with scheme.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8bc49e6..dfb723e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -71,6 +71,17 @@
+
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively. This
+ is most useful for making RM listen to all interfaces by setting to 0.0.0.0.
+
+ yarn.resourcemanager.bind-host
+
+
+
+
The number of threads used to handle applications manager requests.
yarn.resourcemanager.client.thread-count
50
@@ -627,6 +638,17 @@
+
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively. This is
+ most useful for making NM listen to all interfaces by setting to 0.0.0.0.
+
+ yarn.nodemanager.bind-host
+
+
+
+
Environment variables that should be forwarded from the NodeManager's environment to the container's.
yarn.nodemanager.admin-env
MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX
@@ -1164,6 +1186,18 @@
+
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively.
+ This is most useful for making the service listen to all interfaces by setting to
+ 0.0.0.0.
+
+ yarn.timeline-service.bind-host
+
+
+
+
Store class name for timeline store.
yarn.timeline-service.store-class
org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
index 671d1e2..319917c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
@@ -20,10 +20,14 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.InetSocketAddress;
import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.Test;
@@ -132,6 +136,204 @@ public void testRPCRuntimeExceptionUnwrapping() {
Assert.assertTrue(NullPointerException.class.isInstance(t));
Assert.assertTrue(t.getMessage().contains(message));
}
+
+ @Test
+ public void testGetSocketAddr() throws Exception {
+
+ Configuration conf;
+ InetSocketAddress resourceTrackerAddress;
+
+ //all default
+ conf = new Configuration();
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //with address
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.1");
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.1",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //address and socket
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5001");
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.2",
+ 5001),
+ resourceTrackerAddress);
+
+ //bind host only
+ conf = new Configuration();
+ conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.3");
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.3",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //bind host and address no port
+ conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2");
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "0.0.0.0",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //bind host and address with port
+ conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5003");
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "0.0.0.0",
+ 5003),
+ resourceTrackerAddress);
+
+ }
+
+ @Test
+ public void testUpdateConnectAddr() throws Exception {
+ Configuration conf;
+ InetSocketAddress resourceTrackerConnectAddress;
+ InetSocketAddress serverAddress;
+
+ //specify address, default port
+ conf = new Configuration();
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.1");
+ serverAddress = new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+
+ resourceTrackerConnectAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ serverAddress);
+
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.1",
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])),
+ resourceTrackerConnectAddress);
+
+ //specify address, specify port, server port wins
+ conf = new Configuration();
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5002");
+ serverAddress = new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+
+ resourceTrackerConnectAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ serverAddress);
+
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.2",
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])),
+ resourceTrackerConnectAddress);
+
+ //override port from server, but verify that server address (may be set from bind host)
+ //is ignored, and so is any bind-host address which might be specified
+ conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.3");
+ serverAddress = new InetSocketAddress(
+ "10.0.0.3",
+ 5003);
+ resourceTrackerConnectAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ serverAddress);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.2",
+ 5003),
+ resourceTrackerConnectAddress);
+ }
+
+ @Test
+ public void testGetAddressAsString() throws Exception {
+ Configuration conf;
+ String address;
+
+ //defaults
+ conf = new Configuration();
+ address = RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ assertEquals(YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, address);
+
+ //with address
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "10.0.0.1:5001");
+ address = RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ assertEquals("10.0.0.1:5001", address);
+
+ //with bind host, port from address
+ conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.2");
+ address = RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ assertEquals("10.0.0.2:5001", address);
+
+ //with bind host address without port
+ conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.3");
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "10.0.0.1");
+ address = RPCUtil.getAddressAsString(conf,
+ YarnConfiguration.RM_BIND_HOST,
+ conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ assertEquals("10.0.0.3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT, address);
+
+ }
private void verifyRemoteExceptionUnwrapping(
Class extends Throwable> expectedLocalException,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e15198b..4956620 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -56,6 +56,7 @@
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
public class ApplicationHistoryClientService extends AbstractService {
@@ -75,10 +76,12 @@ public ApplicationHistoryClientService(ApplicationHistoryManager history) {
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address =
- conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
+ InetSocketAddress address = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
server =
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
@@ -87,9 +90,10 @@ protected void serviceStart() throws Exception {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT));
server.start();
- this.bindAddress =
- conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
- server.getListenerAddress());
+ this.bindAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ server.getListenerAddress());
LOG.info("Instantiated ApplicationHistoryClientService at "
+ this.bindAddress);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 02a3bb1..40e2d08 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -192,7 +192,7 @@ protected void startWebApp() {
TimelineAuthenticationFilterInitializer.class.getName()
+ initializers);
}
- String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ String bindAddress = WebAppUtils.getAHSWebAppBindURLWithoutScheme(conf);
LOG.info("Instantiating AHSWebApp at " + bindAddress);
try {
AHSWebApp ahsWebApp = AHSWebApp.getInstance();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 1e155d2..1c7fce7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -274,7 +274,9 @@ protected void serviceStart() throws Exception {
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress initialAddress = conf.getSocketAddr(
+ InetSocketAddress initialAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.NM_BIND_HOST,
YarnConfiguration.NM_ADDRESS,
YarnConfiguration.DEFAULT_NM_ADDRESS,
YarnConfiguration.DEFAULT_NM_PORT);
@@ -296,14 +298,28 @@ protected void serviceStart() throws Exception {
" server is still starting.");
this.setBlockNewContainerRequests(true);
server.start();
- InetSocketAddress connectAddress = NetUtils.getConnectAddress(server);
+
+ InetSocketAddress connectAddress;
+ String connectHost = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
+ if (connectHost == null || connectHost.isEmpty()) {
+ // Get hostname and port from the listening endpoint.
+ connectAddress = NetUtils.getConnectAddress(server);
+ } else {
+ // Combine the configured hostname with the port from the listening
+ // endpoint. This gets the correct port number if the configuration
+ // specifies an ephemeral port (port number 0).
+ connectAddress = NetUtils.getConnectAddress(
+ new InetSocketAddress(connectHost.split(":")[0],
+ server.getListenerAddress().getPort()));
+ }
+
NodeId nodeId = NodeId.newInstance(
connectAddress.getAddress().getCanonicalHostName(),
connectAddress.getPort());
((NodeManager.NMContext)context).setNodeId(nodeId);
this.context.getNMTokenSecretManager().setNodeId(nodeId);
this.context.getContainerTokenSecretManager().setNodeId(nodeId);
- LOG.info("ContainerManager started at " + connectAddress);
+ LOG.info("ContainerManager started at " + initialAddress);
super.serviceStart();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 554b368..794d108 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -81,6 +81,7 @@
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
@@ -250,7 +251,8 @@ public void serviceInit(Configuration conf) throws Exception {
conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
cacheCleanupPeriod =
conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
- localizationServerAddress = conf.getSocketAddr(
+ localizationServerAddress = RPCUtil.getSocketAddr(conf,
+ YarnConfiguration.NM_BIND_HOST,
YarnConfiguration.NM_LOCALIZER_ADDRESS,
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
@@ -340,9 +342,11 @@ public void serviceStart() throws Exception {
cacheCleanupPeriod, cacheCleanupPeriod, TimeUnit.MILLISECONDS);
server = createServer();
server.start();
- localizationServerAddress =
- getConfig().updateConnectAddr(YarnConfiguration.NM_LOCALIZER_ADDRESS,
- server.getListenerAddress());
+ localizationServerAddress = RPCUtil.updateConnectAddr(
+ getConfig(),
+ YarnConfiguration.NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
+ server.getListenerAddress());
LOG.info("Localizer started on port " + server.getPort());
super.serviceStart();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
index 2f78ec4..04d68d8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
@@ -55,7 +55,7 @@ public WebServer(Context nmContext, ResourceView resourceView,
@Override
protected void serviceStart() throws Exception {
- String bindAddress = WebAppUtils.getNMWebAppURLWithoutScheme(getConfig());
+ String bindAddress = WebAppUtils.getNMWebAppBindURLWithoutScheme(getConfig());
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 1d2f376..cda3805 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -90,7 +90,9 @@
private EmbeddedElectorService embeddedElector;
private Server server;
- private InetSocketAddress masterServiceAddress;
+
+ // Address to use for binding. May be a wildcard address.
+ private InetSocketAddress masterServiceBindAddress;
private AccessControlList adminAcl;
private final RecordFactory recordFactory =
@@ -114,10 +116,13 @@ public void serviceInit(Configuration conf) throws Exception {
}
}
- masterServiceAddress = conf.getSocketAddr(
+ masterServiceBindAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
+
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
@@ -141,7 +146,7 @@ protected void startServer() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server = (Server) rpc.getServer(
- ResourceManagerAdministrationProtocol.class, this, masterServiceAddress,
+ ResourceManagerAdministrationProtocol.class, this, masterServiceBindAddress,
conf, null,
conf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));
@@ -170,7 +175,9 @@ protected void startServer() throws Exception {
}
this.server.start();
- conf.updateConnectAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
+ RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
server.getListenerAddress());
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index e60add4..61132c8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -126,7 +126,9 @@ protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress masterServiceAddress = conf.getSocketAddr(
+ InetSocketAddress masterServiceAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
@@ -158,9 +160,10 @@ protected void serviceStart() throws Exception {
}
this.server.start();
- this.bindAddress =
- conf.updateConnectAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- server.getListenerAddress());
+ this.bindAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ server.getListenerAddress());
super.serviceStart();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 9743760..96502e7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -197,10 +197,13 @@ protected void serviceStart() throws Exception {
}
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
}
-
+
this.server.start();
- clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_ADDRESS,
- server.getListenerAddress());
+ clientBindAddress = RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ server.getListenerAddress());
+
super.serviceStart();
}
@@ -213,7 +216,10 @@ protected void serviceStop() throws Exception {
}
InetSocketAddress getBindAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ return RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 7dceda2..677a517 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -150,7 +150,8 @@
private AppReportFetcher fetcher = null;
protected ResourceTrackerService resourceTracker;
- private String webAppAddress;
+ @VisibleForTesting
+ protected String webAppAddress;
private ConfigurationProvider configurationProvider = null;
/** End of Active services */
@@ -225,7 +226,7 @@ protected void serviceInit(Configuration conf) throws Exception {
}
createAndInitActiveServices();
- webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(this.conf);
+ webAppAddress = WebAppUtils.getRMWebAppBindURLWithoutScheme(this.conf);
this.rmLoginUGI = UserGroupInformation.getCurrentUser();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f2a8376..c114f4b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -42,6 +42,7 @@
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -120,7 +121,9 @@ public ResourceTrackerService(RMContext rmContext,
@Override
protected void serviceInit(Configuration conf) throws Exception {
- resourceTrackerAddress = conf.getSocketAddr(
+ resourceTrackerAddress = RPCUtil.getSocketAddr(
+ conf,
+ YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
@@ -177,8 +180,10 @@ protected void serviceStart() throws Exception {
}
this.server.start();
- conf.updateConnectAddr(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
- server.getListenerAddress());
+ RPCUtil.updateConnectAddr(conf,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ server.getListenerAddress());
}
@Override
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index 610023b..0de979a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -380,7 +380,19 @@ public void testHAIDLookup() {
}
@Test
- public void testHAWithRMHostName() {
+ public void testHAWithRMHostName() throws Exception {
+ innerTestHAWithRMHostName(false);
+ configuration.clear();
+ setUp();
+ innerTestHAWithRMHostName(true);
+ }
+
+ public void innerTestHAWithRMHostName(boolean includeBindHost) {
+ //this is run two times, with and without a bind host configured
+ if (includeBindHost) {
+ configuration.set(YarnConfiguration.RM_BIND_HOST, "9.9.9.9");
+ }
+
//test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set
//We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration
configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
@@ -400,6 +412,16 @@ public void testHAWithRMHostName() {
RM2_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM2_NODE_ID)));
assertEquals("RPC address not set for " + confKey,
RM3_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM3_NODE_ID)));
+ if (includeBindHost) {
+ assertEquals("Web address misconfigured WITH bind-host",
+ rm.webAppAddress.substring(0, 7), "9.9.9.9");
+ } else {
+ //amazingly, RM Webapp has always been 0.0.0.0 in HA cases, as it ignores the
+ //ha resource manager instance config and looks for the non-ha value, fails to
+ //find it, and uses the default instead
+ assertEquals("Web address misconfigured WITHOUT bind-host",
+ rm.webAppAddress.substring(0, 7), "0.0.0.0");
+ }
}
} catch (YarnRuntimeException e) {
fail("Should not throw any exceptions.");