diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index 8900b16..bf5eeba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -26,13 +26,14 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; -import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -252,4 +253,38 @@ private void verifyExpectedException(String exceptionMessage){ .contains("Application with id '" + fakeAppId + "' " + "doesn't exist in RM.")); } + + @Test + public void testRMWebAppFailOver() throws YarnException, + InterruptedException, IOException { + cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); + + cluster.init(conf); + cluster.start(); + getAdminService(0).transitionToActive(req); + String rm1Url = "http://0.0.0.0:18088"; + String rm2Url = "http://0.0.0.0:28088"; + String header = getHeader("Refresh", rm2Url); + assertTrue(header.contains("; url=" + rm1Url)); + + header = getHeader("Refresh", rm2Url + "/cluster/cluster"); + assertEquals(null, header); + + // Due to the limitation of MiniYARNCluster, we couldn't test the case after + // explicitFailover(); + } + + static String getHeader(String field, String url) { + String fieldHeader = null; + try { + Map> map = + new URL(url).openConnection().getHeaderFields(); + fieldHeader = map.get(field).get(0); + } catch (Exception e) { + // throw new RuntimeException(e); + } + return fieldHeader; + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java index 66dd21b..a7aded9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java @@ -1,26 +1,27 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.webapp; import static com.google.common.base.Preconditions.checkState; import java.io.IOException; +import java.io.PrintWriter; import java.util.Timer; import java.util.TimerTask; @@ -89,6 +90,23 @@ public void service(HttpServletRequest req, HttpServletResponse res) prepareToExit(); return; } + + if (webApp.isStandbyMode() && !uri.equals("/" + webApp.name() + "/cluster")) { + String redirectPath = webApp.getRedirectPath(); + if (redirectPath != null && !redirectPath.isEmpty()) { + String redirectMsg = + "This is a standby resource manager, redirecting to the current active one: " + + redirectPath; + res.addHeader("Refresh", "3; url=" + redirectPath); + PrintWriter out = res.getWriter(); + out.println("Redirecting..." + redirectMsg); + webApp.setRedirectPath("/" + webApp.name()); + return; + } + webApp.setRedirectPath("/" + webApp.name()); + + } + // if they provide a redirectPath go there instead of going to // "/" so that filters can differentiate the webapps. if (uri.equals("/")) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index 90323ee..580aa31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -47,15 +47,17 @@ /** * @see WebApps for a usage example */ -@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) +@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) public abstract class WebApp extends ServletModule { private static final Logger LOG = LoggerFactory.getLogger(WebApp.class); - public enum HTTP { GET, POST, HEAD, PUT, DELETE }; + public enum HTTP { + GET, POST, HEAD, PUT, DELETE + }; private volatile String name; private volatile List servePathSpecs = new ArrayList(); - // path to redirect to if user goes to "/" + // path to redirect to if user goes to "/", or if redirect to active rm private volatile String redirectPath; private volatile String wsName; private volatile Configuration conf; @@ -110,35 +112,63 @@ public void joinThread() { } } - void setConf(Configuration conf) { this.conf = conf; } + void setConf(Configuration conf) { + this.conf = conf; + } - @Provides public Configuration conf() { return conf; } + @Provides + public Configuration conf() { + return conf; + } - @Provides Router router() { return router; } + @Provides + Router router() { + return router; + } - @Provides WebApp webApp() { return this; } + @Provides + WebApp webApp() { + return this; + } - void setName(String name) { this.name = name; } + void setName(String name) { + this.name = name; + } - public String name() { return this.name; } + public String name() { + return this.name; + } - void addServePathSpec(String path) { this.servePathSpecs.add(path); } + void addServePathSpec(String path) { + this.servePathSpecs.add(path); + } public String[] getServePathSpecs() { return this.servePathSpecs.toArray(new String[this.servePathSpecs.size()]); } /** - * Set a path to redirect the user to if they just go to "/". For - * instance "/" goes to "/yarn/apps". This allows the filters to - * more easily differentiate the different webapps. - * @param path the path to redirect to + * Set a path to redirect the user to if they just go to "/". For instance "/" + * goes to "/yarn/apps". This allows the filters to more easily differentiate + * the different webapps. + * + * @param path the path to redirect to */ - void setRedirectPath(String path) { this.redirectPath = path; } + protected void setRedirectPath(String path) { + this.redirectPath = path; + } + + void setWebServices(String name) { + this.wsName = name; + } - void setWebServices (String name) { this.wsName = name; } + public String wsName() { + return this.wsName; + } - public String getRedirectPath() { return this.redirectPath; } + public String getRedirectPath() { + return this.redirectPath; + } void setHostClass(Class cls) { router.setHostClass(cls); @@ -267,5 +297,10 @@ static String getPrefix(String pathSpec) { return pathSpec.substring(start, ci + 1); } + public boolean isStandbyMode() { + return false; + } + public abstract void setup(); + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 5a0980e..f15e5e6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -1,26 +1,34 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMHAUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -34,6 +42,7 @@ public class RMWebApp extends WebApp implements YarnWebParams { private final ResourceManager rm; + private static final Log LOG = LogFactory.getLog(RMWebApp.class.getName()); public RMWebApp(ResourceManager rm) { this.rm = rm; @@ -58,5 +68,44 @@ public void setup() { route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route("/scheduler", RmController.class, "scheduler"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); + + } + + @Override + public boolean isStandbyMode() { + String path; + if (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY) { + path = findRedirectPath((YarnConfiguration) rm.getConfig()); + if ("".equals(path)) { + setRedirectPath(path); + } else { + setRedirectPath("http://" + path); + } + return true; + } else + return false; + } + + private String findRedirectPath(YarnConfiguration yarnConf) { + String activeRMHAId = RMHAUtils.findActiveRMHAId(yarnConf); + String rmId = yarnConf.get(YarnConfiguration.RM_HA_ID); + if (activeRMHAId != null) { + yarnConf.set(YarnConfiguration.RM_HA_ID, activeRMHAId); + InetSocketAddress sock; + if (!HttpConfig.isSecure()) { + sock = + yarnConf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + } else { + sock = + yarnConf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); + } + yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); + return sock.getHostName() + ":" + Integer.toString(sock.getPort()); + } + return ""; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java new file mode 100644 index 0000000..92127b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.util.Collection; + +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.ha.HAServiceTarget; +import org.apache.hadoop.yarn.client.RMHAServiceTarget; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class RMHAUtils { + + public static String findActiveRMHAId(YarnConfiguration yarnConf) { + String rmId = yarnConf.get(YarnConfiguration.RM_HA_ID); + Collection rmIds = + yarnConf.getStringCollection(YarnConfiguration.RM_HA_IDS); + for (String currentId : rmIds) { + yarnConf.set(YarnConfiguration.RM_HA_ID, currentId); + try { + HAServiceState haState = getHAState(yarnConf); + if (haState.equals(HAServiceState.ACTIVE)) { + yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); + return currentId; + } + } catch (Exception e) { + } + } + yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); + return null; + } + + private static HAServiceState getHAState(YarnConfiguration yarnConf) + throws Exception { + HAServiceTarget haServiceTarget; + + haServiceTarget = new RMHAServiceTarget(yarnConf); + int rpcTimeoutForChecks = -1; + if (yarnConf != null) { + rpcTimeoutForChecks = + yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, + CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); + } + HAServiceProtocol proto = + haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); + HAServiceState haState = proto.getServiceStatus().getState(); + return haState; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java new file mode 100644 index 0000000..92127b5 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAUtils.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.util.Collection; + +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.ha.HAServiceTarget; +import org.apache.hadoop.yarn.client.RMHAServiceTarget; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +public class RMHAUtils { + + public static String findActiveRMHAId(YarnConfiguration yarnConf) { + String rmId = yarnConf.get(YarnConfiguration.RM_HA_ID); + Collection rmIds = + yarnConf.getStringCollection(YarnConfiguration.RM_HA_IDS); + for (String currentId : rmIds) { + yarnConf.set(YarnConfiguration.RM_HA_ID, currentId); + try { + HAServiceState haState = getHAState(yarnConf); + if (haState.equals(HAServiceState.ACTIVE)) { + yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); + return currentId; + } + } catch (Exception e) { + } + } + yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); + return null; + } + + private static HAServiceState getHAState(YarnConfiguration yarnConf) + throws Exception { + HAServiceTarget haServiceTarget; + + haServiceTarget = new RMHAServiceTarget(yarnConf); + int rpcTimeoutForChecks = -1; + if (yarnConf != null) { + rpcTimeoutForChecks = + yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, + CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); + } + HAServiceProtocol proto = + haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); + HAServiceState haState = proto.getServiceStatus().getState(); + return haState; + } + +}