diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java index a48b3c0..e6f23f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java @@ -178,8 +178,5 @@ private static void uappend(StringBuilder sb, String part) { public static String percent(double value) { return String.format("%.2f", value * 100); } - - public static String getPartUrl(String url, String part) { - return url.substring(url.indexOf(part)); - } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 61ac708..6c3a420 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.yarn.webapp.util; -import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -29,9 +29,7 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.ConverterUtils; @Private @Evolving @@ -169,18 +167,22 @@ public static String getURLWithScheme(String schemePrefix, String url) { } } - public static String getLogUrl(String nodeHttpAddress, String allocatedNode, - ContainerId containerId, String user) { - return join("//", nodeHttpAddress, "/logs", "/", - allocatedNode, "/", ConverterUtils.toString(containerId), "/", - ConverterUtils.toString(containerId), "/", user); + public static String getRunningLogURL( + String nodeHttpAddress, String containerId, String user) { + return PATH_JOINER.join( + nodeHttpAddress, "node", "containerlogs", containerId, user); + } + + public static String getAggregatedLogURL( + String allocatedNode, String containerId, String entity, String user) { + return PATH_JOINER.join(allocatedNode, containerId, entity, user); } /** * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on * the configuration. * - * @return the schmeme (HTTP / HTTPS) + * @return the scheme (HTTP / HTTPS) */ public static String getHttpSchemePrefix(Configuration conf) { return YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 2040f57..04177e7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; + import java.util.Collections; import java.util.Set; @@ -31,6 +33,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -41,6 +44,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -49,9 +53,12 @@ @Path("/ws/v1/applicationhistory") public class AHSWebServices extends WebServices { + private String serverHttpAddress; + @Inject - public AHSWebServices(ApplicationContext appContext) { - super(appContext); + public AHSWebServices(ApplicationContext appContext, Configuration conf) { + super(appContext, conf); + this.serverHttpAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf); } @GET @@ -124,7 +131,16 @@ public ContainersInfo getContainers(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("appid") String appId, @PathParam("appattemptid") String appAttemptId) { init(res); - return super.getContainers(req, res, appId, appAttemptId); + ContainersInfo containers = + super.getContainers(req, res, appId, appAttemptId); + for (ContainerInfo container : containers.getContainers()) { + if (container.getLogUrl() != null) { + container.setLogUrl(PATH_JOINER.join(req.getScheme() + "://" + + serverHttpAddress, "applicationhistory", "logs", + container.getLogUrl())); + } + } + return containers; } @GET @@ -136,7 +152,14 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, @PathParam("appattemptid") String appAttemptId, @PathParam("containerid") String containerId) { init(res); - return super.getContainer(req, res, appId, appAttemptId, containerId); + ContainerInfo container = + super.getContainer(req, res, appId, appAttemptId, containerId); + if (container.getLogUrl() != null) { + container.setLogUrl(PATH_JOINER.join(req.getScheme() + "://" + + serverHttpAddress, "applicationhistory", "logs", + container.getLogUrl())); + } + return container; } private static void diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java index 81e52d9..ee38830 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java @@ -78,7 +78,7 @@ protected void writeContainerStartData(ContainerId containerId) protected void writeContainerFinishData(ContainerId containerId) throws IOException { store.containerFinished(ContainerFinishData.newInstance(containerId, 0, - containerId.toString(), "http://localhost:0/log", 0, + containerId.toString(), "node/containerid/entity/user", 0, ContainerState.COMPLETE)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index fa759bf..d2981cb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -25,6 +25,7 @@ import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -34,12 +35,14 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -287,7 +290,11 @@ public void testSingleContainer() throws Exception { container.getString("assignedNodeId")); assertEquals(Priority.newInstance(containerId.getId()).toString(), container.getString("priority")); - assertEquals("http://localhost:0/log", container.getString("logUrl")); + Configuration conf = new YarnConfiguration(); + assertEquals( WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf) + + "/applicationhistory/logs/node/containerid/entity/user", + container.getString("logUrl")); assertEquals(ContainerState.COMPLETE.toString(), container.getString("containerState")); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 4bde1a3..5ac902a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID; import java.io.IOException; @@ -27,6 +27,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -37,6 +38,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -45,11 +47,18 @@ public class AppAttemptBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class); - private final ApplicationContext appContext; + protected ApplicationContext appContext; + protected Configuration conf; + protected String serverHttpAddress; @Inject - public AppAttemptBlock(ApplicationContext appContext) { + public AppAttemptBlock(ApplicationContext appContext, Configuration conf, + ViewContext ctx) { + super(ctx); this.appContext = appContext; + this.conf = conf; + this.serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); } @Override @@ -127,9 +136,12 @@ protected void render(Block html) { StringBuilder containersTableData = new StringBuilder("[\n"); for (ContainerReport containerReport : containers) { - String logURL = containerReport.getLogUrl(); - logURL = getPartUrl(logURL, "log"); ContainerInfo container = new ContainerInfo(containerReport); + if (container.getLogUrl() != null) { + container.setLogUrl(PATH_JOINER.join( + serverHttpAddress, "applicationhistory", "logs", + container.getLogUrl())); + } // ConatinerID numerical value parsed by parseHadoopID in // yarn.dt.plugins.js containersTableData @@ -144,8 +156,10 @@ protected void render(Block html) { StringEscapeUtils.escapeJavaScript(StringEscapeUtils .escapeHtml(container.getAssignedNodeId()))).append("\",\"") .append(container.getContainerExitStatus()).append("\",\"") - .append(logURL == null ? "N/A" : "Logs").append("\"],\n"); + .append(container.getLogUrl() == null ? + "#" : container.getLogUrl()).append("'>") + .append(container.getLogUrl() == null ? + "N/A" : "Logs").append("\"],\n"); } if (containersTableData.charAt(containersTableData.length() - 2) == ',') { containersTableData.delete(containersTableData.length() - 2, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 09bb958..ee245de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -19,14 +19,14 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import java.io.IOException; import java.util.Collection; import org.apache.commons.lang.StringEscapeUtils; -import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -49,11 +50,17 @@ public class AppBlock extends HtmlBlock { protected ApplicationContext appContext; + protected Configuration conf; + protected String serverHttpAddress; @Inject - AppBlock(ApplicationContext appContext, ViewContext ctx) { + AppBlock(ApplicationContext appContext, Configuration conf, + ViewContext ctx) { super(ctx); this.appContext = appContext; + this.conf = conf; + this.serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); } @Override @@ -145,7 +152,10 @@ protected void render(Block html) { ContainerInfo container = new ContainerInfo(containerReport); startTime = container.getStartedTime(); logsLink = containerReport.getLogUrl(); - logsLink = getPartUrl(logsLink, "log"); + if (logsLink != null) { + logsLink = PATH_JOINER.join(serverHttpAddress, "applicationhistory", + "logs", logsLink); + } } String nodeLink = null; if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0 @@ -169,8 +179,8 @@ protected void render(Block html) { nodeLink == null ? "N/A" : StringEscapeUtils .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink))) .append("\",\"") - .append(nodeLink == null ? "N/A" : "Logs").append("\"],\n"); + .append(logsLink == null ? "#" : logsLink).append("'>") + .append(logsLink == null ? "N/A" : "Logs").append("\"],\n"); } if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { attemptsTableData.delete(attemptsTableData.length() - 2, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index d4a77a8..b1fe57b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -28,6 +28,7 @@ import java.util.HashSet; import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.api.ApplicationContext; @@ -42,11 +43,14 @@ public class AppsBlock extends HtmlBlock { protected ApplicationContext appContext; + protected Configuration conf; @Inject - AppsBlock(ApplicationContext appContext, ViewContext ctx) { + AppsBlock(ApplicationContext appContext, Configuration conf, + ViewContext ctx) { super(ctx); this.appContext = appContext; + this.conf = conf; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java index 0b1d27d..709c6f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java @@ -18,13 +18,14 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -40,12 +42,18 @@ public class ContainerBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(ContainerBlock.class); - private final ApplicationContext appContext; + protected ApplicationContext appContext; + protected Configuration conf; + protected String serverHttpAddress; @Inject - public ContainerBlock(ApplicationContext appContext, ViewContext ctx) { + public ContainerBlock(ApplicationContext appContext, Configuration conf, + ViewContext ctx) { super(ctx); this.appContext = appContext; + this.conf = conf; + this.serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); } @Override @@ -79,8 +87,11 @@ protected void render(Block html) { } ContainerInfo container = new ContainerInfo(containerReport); - String logURL = containerReport.getLogUrl(); - logURL = getPartUrl(logURL, "log"); + if (container.getLogUrl() != null) { + container.setLogUrl(PATH_JOINER.join( + serverHttpAddress, "applicationhistory", "logs", + container.getLogUrl())); + } setTitle(join("Container ", containerid)); info("Container Overview") @@ -97,7 +108,8 @@ protected void render(Block html) { "Resource:", container.getAllocatedMB() + " Memory, " + container.getAllocatedVCores() + " VCores") - ._("Logs:", logURL == null ? "#" : url(logURL), "Logs") + ._("Logs:", container.getLogUrl() == null ? "#" : container.getLogUrl(), + container.getLogUrl() == null ? "N/A" : "Logs") ._("Diagnostics:", container.getDiagnosticsInfo()); html._(InfoBlock.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 5a82551..c6e79c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -28,6 +28,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.WebApplicationException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -50,9 +51,11 @@ public class WebServices { protected ApplicationContext appContext; + protected Configuration conf; - public WebServices(ApplicationContext appContext) { + public WebServices(ApplicationContext appContext, Configuration conf) { this.appContext = appContext; + this.conf = conf; } public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java index 2e4436e..bc202d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java @@ -106,6 +106,10 @@ public String getLogUrl() { return logUrl; } + public void setLogUrl(String logUrl) { + this.logUrl = logUrl; + } + public int getContainerExitStatus() { return containerExitStatus; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 57fb703..a6bdad9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -164,6 +165,7 @@ public RMContainerImpl(Container container, this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); + this.logURL = null; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -363,9 +365,9 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { public void transition(RMContainerImpl container, RMContainerEvent event) { // The logs of running containers should be found on NM webUI // The logs should be accessible after the container is launched - container.logURL = WebAppUtils.getLogUrl(container.container - .getNodeHttpAddress(), container.getAllocatedNode().toString(), - container.containerId, container.user); + container.logURL = + WebAppUtils.getRunningLogURL("//" + container.container.getNodeHttpAddress(), + ConverterUtils.toString(container.containerId), container.user); // Unregister from containerAllocationExpirer. container.containerAllocationExpirer.unregister(container .getContainerId()); @@ -380,9 +382,15 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { container.finishTime = System.currentTimeMillis(); container.finishedStatus = finishedEvent.getRemoteContainerStatus(); - // TODO: when AHS webUI is ready, logURL should be updated to point to - // the web page that will show the aggregated logs - + // when AHS webUI is ready, logURL should be updated to point to + // the web page that will show the aggregated logs, if the container + // has been launched before + if (container.getState() == RMContainerState.RUNNING) { + String containerId = ConverterUtils.toString(container.containerId); + container.logURL = WebAppUtils.getAggregatedLogURL( + container.getAllocatedNode().toString(), containerId, containerId, + container.user); + } // Inform AppAttempt container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus())); @@ -396,6 +404,12 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { FinishedTransition { @Override public void transition(RMContainerImpl container, RMContainerEvent event) { + // Given the container is finished too fast, we still need to record the + // log url for the history purpose. + String containerId = ConverterUtils.toString(container.containerId); + container.logURL = WebAppUtils.getAggregatedLogURL( + container.getAllocatedNode().toString(), containerId, containerId, + container.user); // Unregister from containerAllocationExpirer. container.containerAllocationExpirer.unregister(container diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 35f21db..0bb9cf7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import static org.apache.hadoop.yarn.util.StringHelper.join; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -26,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @XmlRootElement(name = "appAttempt") @XmlAccessorType(XmlAccessType.FIELD) @@ -55,10 +54,9 @@ public AppAttemptInfo(RMAppAttempt attempt, String user) { this.containerId = masterContainer.getId().toString(); this.nodeHttpAddress = masterContainer.getNodeHttpAddress(); this.nodeId = masterContainer.getNodeId().toString(); - this.logsLink = join("//", - masterContainer.getNodeHttpAddress(), - "/node", "/containerlogs/", - ConverterUtils.toString(masterContainer.getId()), "/", user); + this.logsLink = + WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(), + ConverterUtils.toString(masterContainer.getId()), user); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 6cb6114..11f798d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import static org.apache.hadoop.yarn.util.StringHelper.join; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; -import com.google.common.base.Joiner; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; @@ -38,6 +35,8 @@ import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import com.google.common.base.Joiner; + @XmlRootElement(name = "app") @XmlAccessorType(XmlAccessType.FIELD) public class AppInfo { @@ -131,12 +130,10 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) { Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; - String url = join(schemePrefix, - masterContainer.getNodeHttpAddress(), - "/node", "/containerlogs/", + this.amContainerLogs = WebAppUtils.getRunningLogURL( + schemePrefix + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), - "/", app.getUser()); - this.amContainerLogs = url; + app.getUser()); this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index f75f9b0..7e35fd0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -104,8 +104,7 @@ public void testReleaseWhileRunning() { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals( - "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. @@ -137,6 +136,9 @@ public void testReleaseWhileRunning() { .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); + assertEquals( + "host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + rmContainer.getLogURL()); } @Test @@ -191,8 +193,7 @@ public void testExpireWhileRunning() { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals( - "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions.