diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index 3cbde93..242a971 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -21,10 +21,13 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import java.net.InetSocketAddress; import java.util.Collection; @InterfaceAudience.Private @@ -108,8 +111,7 @@ private static void verifyAndSetRMHAIdsList(Configuration conf) { String errmsg = iae.getMessage(); if (confKey == null) { // Error at addSuffix - errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, - getRMHAId(conf)); + errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id); } throwBadConfigurationException(errmsg); } @@ -122,10 +124,11 @@ private static void verifyAndSetRMHAIdsList(Configuration conf) { } private static void verifyAndSetCurrentRMHAId(Configuration conf) { - String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + String rmId = getRMHAId(conf); if (rmId == null) { - throwBadConfigurationException( - getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID)); + String msg = "Configuration " + YarnConfiguration.RM_ADDRESS + + " must be suffixed with RM ID for HA configuration."; + throwBadConfigurationException(msg); } else { Collection ids = getRMHAIds(conf); if (!ids.contains(rmId)) { @@ -179,7 +182,34 @@ public static void verifyAndSetAllServiceAddresses(Configuration conf) { * @return RM Id on success */ public static String getRMHAId(Configuration conf) { - return conf.get(YarnConfiguration.RM_HA_ID); + int found = 0; + String currentRMId = conf.getTrimmed(YarnConfiguration.RM_HA_ID); + if(currentRMId == null) { + for(String rmId : getRMHAIds(conf)) { + String key = addSuffix(YarnConfiguration.RM_ADDRESS, rmId); + String addr = conf.get(key); + if (addr == null) { + continue; + } + InetSocketAddress s; + try { + s = NetUtils.createSocketAddr(addr); + } catch (Exception e) { + LOG.warn("Exception in creating socket address " + addr, e); + continue; + } + if (!s.isUnresolved() && NetUtils.isLocalAddress(s.getAddress())) { + currentRMId = rmId.trim(); + found++; + } + } + } + if (found > 1) { // Only one address must match the local address + String msg = "The HA Configuration has multiple addresses that match " + + "local node's address."; + throw new HadoopIllegalArgumentException(msg); + } + return currentRMId; } @VisibleForTesting diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c493742..73698dc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -395,7 +395,9 @@ the Active mode when prompted to. (2) The nodes in the RM ensemble are listed in yarn.resourcemanager.ha.rm-ids - (3) The id of each RM comes from yarn.resourcemanager.ha.id + (3) The id of each RM either comes from yarn.resourcemanager.ha.id + if yarn.resourcemanager.ha.id is explicitly specified or can be + figured out by matching yarn.resourcemanager.address.{id} with local address (4) The actual physical addresses come from the configs of the pattern - {rpc-config}.{id} yarn.resourcemanager.ha.enabled @@ -442,7 +444,10 @@ The id (string) of the current RM. When HA is enabled, this - is a required config. See description of yarn.resourcemanager.ha.enabled + is an optional config. The id of current RM can be set by explicitly + specifying yarn.resourcemanager.ha.id or figured out by matching + yarn.resourcemanager.address.{id} with local address + See description of yarn.resourcemanager.ha.enabled for full details on how this is used. yarn.resourcemanager.ha.id diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index 9d1a467..0d81db4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -48,10 +48,10 @@ private static final String STATE_ERR = "ResourceManager is in wrong HA state"; - private static final String RM1_ADDRESS = "0.0.0.0:0"; + private static final String RM1_ADDRESS = "1.1.1.1:1"; private static final String RM1_NODE_ID = "rm1"; - private static final String RM2_ADDRESS = "1.1.1.1:1"; + private static final String RM2_ADDRESS = "0.0.0.0:0"; private static final String RM2_NODE_ID = "rm2"; @Before @@ -62,7 +62,6 @@ public void setUp() throws Exception { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } - configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); } private void checkMonitorHealth() throws IOException { @@ -278,6 +277,22 @@ protected Dispatcher createDispatcher() { rm.stop(); } + @Test + public void testHAIDLookup() { + //test implicitly lookup HA-ID + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), "rm2"); + + //test explicitly lookup HA-ID + configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + conf = new YarnConfiguration(configuration); + rm.init(conf); + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), "rm1"); + } + @SuppressWarnings("rawtypes") class MyCountingDispatcher extends AbstractService implements Dispatcher {