diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java index 3c72568..15c3680 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java @@ -100,11 +100,22 @@ public static FinishApplicationMasterRequest newInstance( public abstract String getTrackingUrl(); /** - * Set the tracking URLfor the ApplicationMaster - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param url tracking URLfor the - * ApplicationMaster + * Set the final tracking URLfor the ApplicationMaster. + * This is the web-URL to which ResourceManager or web-application proxy will + * redirect client/users once the application is finished and the + * ApplicationMaster is gone. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param url + * tracking URLfor the ApplicationMaster */ @Public @Stable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index 05668dd..0b485d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -112,11 +112,22 @@ public static RegisterApplicationMasterRequest newInstance(String host, public abstract String getTrackingUrl(); /** - * Set the tracking URL for the ApplicationMaster. - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param trackingUrl tracking URL for the - * ApplicationMaster + * Set the tracking URLfor the ApplicationMaster while + * it is running. This is the web-URL to which ResourceManager or + * web-application proxy will redirect client/users while the application and + * the ApplicationMaster are still running. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param trackingUrl + * tracking URLfor the ApplicationMaster */ @Public @Stable diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 530b343..ae3bc9f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -994,7 +994,7 @@ public void transition(RMAppAttemptImpl appAttempt, } } - static final class AMRegisteredTransition extends BaseTransition { + private static final class AMRegisteredTransition extends BaseTransition { @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { @@ -1003,7 +1003,8 @@ public void transition(RMAppAttemptImpl appAttempt, = (RMAppAttemptRegistrationEvent) event; appAttempt.host = registrationEvent.getHost(); appAttempt.rpcPort = registrationEvent.getRpcport(); - appAttempt.origTrackingUrl = registrationEvent.getTrackingurl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(registrationEvent.getTrackingurl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); @@ -1138,7 +1139,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event; appAttempt.diagnostics.append(unregisterEvent.getDiagnostics()); - appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(unregisterEvent.getTrackingUrl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus(); @@ -1292,4 +1294,8 @@ private void removeCredentials(RMAppAttemptImpl appAttempt) { appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttempt.getAppAttemptId()); } + + private static String sanitizeTrackingUrl(String url) { + return (url == null || url.trim().isEmpty()) ? "N/A" : url; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java deleted file mode 100644 index e69d867..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; - -import junit.framework.Assert; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; - -import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; -import org.junit.Test; -import org.mockito.Mockito; - -public class TestRMAppAttemptImpl { - - private void testTrackingUrl(String url, boolean unmanaged) { - ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance - (ApplicationId.newInstance(1, 2), 1); - EventHandler handler = Mockito.mock(EventHandler.class); - Dispatcher dispatcher = Mockito.mock(Dispatcher.class); - Mockito.when(dispatcher.getEventHandler()).thenReturn(handler); - RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher); - - ApplicationSubmissionContext appContext = - Mockito.mock(ApplicationSubmissionContext.class); - Mockito.when(appContext.getUnmanagedAM()).thenReturn(unmanaged); - - RMAppAttemptImpl attempt = new RMAppAttemptImpl(attemptId, rmContext, null, - null, appContext, new YarnConfiguration(), null); - RMAppAttemptRegistrationEvent event = - Mockito.mock(RMAppAttemptRegistrationEvent.class); - Mockito.when(event.getHost()).thenReturn("h"); - Mockito.when(event.getRpcport()).thenReturn(0); - Mockito.when(event.getTrackingurl()).thenReturn(url); - new RMAppAttemptImpl.AMRegisteredTransition().transition(attempt, event); - if (unmanaged) { - Assert.assertEquals(url, attempt.getTrackingUrl()); - } else { - Assert.assertNotSame(url, attempt.getTrackingUrl()); - Assert.assertTrue(attempt.getTrackingUrl().contains( - ProxyUriUtils.PROXY_SERVLET_NAME)); - Assert.assertTrue(attempt.getTrackingUrl().contains( - attemptId.getApplicationId().toString())); - } - } - - @Test - public void testTrackingUrlUnmanagedAM() { - testTrackingUrl("http://foo:8000/x", true); - } - - @Test - public void testTrackingUrlManagedAM() { - testTrackingUrl("bar:8000/x", false); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 056589a..a878be1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -30,14 +30,18 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; @@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -261,8 +267,22 @@ public void tearDown() throws Exception { private String getProxyUrl(RMAppAttempt appAttempt) { - return pjoin(RM_WEBAPP_ADDR, "proxy", - appAttempt.getAppAttemptId().getApplicationId(), ""); + String url = null; + try { + URI trackingUri = + StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null : + ProxyUriUtils + .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl()); + String proxy = WebAppUtils.getProxyHostAndPort(conf); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); + URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, + appAttempt.getAppAttemptId().getApplicationId()); + url = result.toASCIIString().substring( + HttpConfig.getSchemePrefix().length()); + } catch (URISyntaxException ex) { + Assert.fail(); + } + return url; } /** @@ -448,9 +468,9 @@ private void testAppAttemptRunningState(Container container, assertEquals(container, applicationAttempt.getMasterContainer()); assertEquals(host, applicationAttempt.getHost()); assertEquals(rpcPort, applicationAttempt.getRpcPort()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); @@ -468,7 +488,7 @@ private void testAppAttemptFinishingState(Container container, assertEquals(RMAppAttemptState.FINISHING, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); assertEquals(container, applicationAttempt.getMasterContainer()); @@ -487,9 +507,9 @@ private void testAppAttemptFinishedState(Container container, assertEquals(RMAppAttemptState.FINISHED, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), @@ -603,9 +623,7 @@ private void unregisterApplicationAttempt(Container container, trackingUrl, diagnostics); } - - @Test - public void testUnmanagedAMSuccess() { + private void testUnmanagedAMSuccess(String url) { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); // submit AM and check it goes to LAUNCHED state @@ -615,7 +633,7 @@ public void testUnmanagedAMSuccess() { applicationAttempt.getAppAttemptId()); // launch AM - runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true); + runApplicationAttempt(null, "host", 8042, url, true); // complete a container applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent( @@ -623,13 +641,12 @@ public void testUnmanagedAMSuccess() { applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class))); // complete AM - String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; applicationAttempt.handle(new RMAppAttemptUnregistrationEvent( - applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, + applicationAttempt.getAppAttemptId(), url, finalStatus, diagnostics)); - testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1, + testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1, true); } @@ -824,12 +841,42 @@ public void testUnregisterToKilledFinishing() { "Killed by user"); } + @Test + public void testTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess("oldTrackingUrl"); + } @Test - public void testNoTrackingUrl() { + public void testEmptyTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(""); + } + + @Test + public void testNullTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(null); + } + + @Test + public void testManagedAMWithTrackingUrl() { + testTrackingUrlManagedAM("theTrackingUrl"); + } + + @Test + public void testManagedAMWithEmptyTrackingUrl() { + testTrackingUrlManagedAM(""); + } + + @Test + public void testManagedAMWithNullTrackingUrl() { + testTrackingUrlManagedAM(null); + } + + private void testTrackingUrlManagedAM(String url) { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "", false); + runApplicationAttempt(amContainer, "host", 8042, url, false); + unregisterApplicationAttempt(amContainer, + FinalApplicationStatus.SUCCEEDED, url, "Successful"); } @Test @@ -927,4 +974,12 @@ private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) { } } } + + private void verifyUrl(String url1, String url2) { + if (url1 == null || url1.trim().isEmpty()) { + assertEquals("N/A", url2); + } else { + assertEquals(url1, url2); + } + } }