diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java index a48b3c0..e6f23f0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java @@ -178,8 +178,5 @@ private static void uappend(StringBuilder sb, String part) { public static String percent(double value) { return String.format("%.2f", value * 100); } - - public static String getPartUrl(String url, String part) { - return url.substring(url.indexOf(part)); - } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 61ac708..502d4db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -17,11 +17,13 @@ */ package org.apache.hadoop.yarn.webapp.util; -import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -29,9 +31,7 @@ import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.ConverterUtils; @Private @Evolving @@ -169,18 +169,35 @@ public static String getURLWithScheme(String schemePrefix, String url) { } } - public static String getLogUrl(String nodeHttpAddress, String allocatedNode, - ContainerId containerId, String user) { - return join("//", nodeHttpAddress, "/logs", "/", - allocatedNode, "/", ConverterUtils.toString(containerId), "/", - ConverterUtils.toString(containerId), "/", user); + public static String getLogURL(String nodeHttpAddress, String allocatedNode, + String logsDir, String containerId, String entity, String user) { + List sections = new ArrayList(); + if (nodeHttpAddress != null) { + sections.add(nodeHttpAddress); + } + if (allocatedNode != null) { + sections.add(allocatedNode); + } + if (logsDir != null) { + sections.add(logsDir); + } + if (containerId != null) { + sections.add(containerId); + } + if (entity != null) { + sections.add(entity); + } + if (user != null) { + sections.add(user); + } + return PATH_JOINER.join(sections); } /** * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on * the configuration. * - * @return the schmeme (HTTP / HTTPS) + * @return the scheme (HTTP / HTTPS) */ public static String getHttpSchemePrefix(Configuration conf) { return YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 4bde1a3..b178c9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID; import java.io.IOException; @@ -127,8 +126,6 @@ protected void render(Block html) { StringBuilder containersTableData = new StringBuilder("[\n"); for (ContainerReport containerReport : containers) { - String logURL = containerReport.getLogUrl(); - logURL = getPartUrl(logURL, "log"); ContainerInfo container = new ContainerInfo(containerReport); // ConatinerID numerical value parsed by parseHadoopID in // yarn.dt.plugins.js @@ -144,8 +141,10 @@ protected void render(Block html) { StringEscapeUtils.escapeJavaScript(StringEscapeUtils .escapeHtml(container.getAssignedNodeId()))).append("\",\"") .append(container.getContainerExitStatus()).append("\",\"") - .append(logURL == null ? "N/A" : "Logs").append("\"],\n"); + .append(container.getLogUrl() == null ? + "#" : url("logs", container.getLogUrl())).append("'>") + .append(container.getLogUrl() == null ? + "N/A" : "Logs").append("\"],\n"); } if (containersTableData.charAt(containersTableData.length() - 2) == ',') { containersTableData.delete(containersTableData.length() - 2, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 09bb958..9d5a5d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -19,14 +19,12 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import java.io.IOException; import java.util.Collection; import org.apache.commons.lang.StringEscapeUtils; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -145,7 +143,9 @@ protected void render(Block html) { ContainerInfo container = new ContainerInfo(containerReport); startTime = container.getStartedTime(); logsLink = containerReport.getLogUrl(); - logsLink = getPartUrl(logsLink, "log"); + if (logsLink != null) { + logsLink = url("logs", logsLink); + } } String nodeLink = null; if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0 @@ -169,8 +169,8 @@ protected void render(Block html) { nodeLink == null ? "N/A" : StringEscapeUtils .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink))) .append("\",\"") - .append(nodeLink == null ? "N/A" : "Logs").append("\"],\n"); + .append(logsLink == null ? "#" : logsLink).append("'>") + .append(logsLink == null ? "N/A" : "Logs").append("\"],\n"); } if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { attemptsTableData.delete(attemptsTableData.length() - 2, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java index 0b1d27d..85fc76a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.server.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl; +import static org.apache.hadoop.yarn.util.StringHelper.ujoin; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; import java.io.IOException; @@ -79,8 +79,6 @@ protected void render(Block html) { } ContainerInfo container = new ContainerInfo(containerReport); - String logURL = containerReport.getLogUrl(); - logURL = getPartUrl(logURL, "log"); setTitle(join("Container ", containerid)); info("Container Overview") @@ -97,7 +95,9 @@ protected void render(Block html) { "Resource:", container.getAllocatedMB() + " Memory, " + container.getAllocatedVCores() + " VCores") - ._("Logs:", logURL == null ? "#" : url(logURL), "Logs") + ._("Logs:", container.getLogUrl() == null ? + "#" : ujoin("logs", container.getLogUrl()), + container.getLogUrl() == null ? "N/A" : "Logs") ._("Diagnostics:", container.getDiagnosticsInfo()); html._(InfoBlock.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 57fb703..8a79542 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -147,6 +148,7 @@ private Priority reservedPriority; private long startTime; private long finishTime; + private boolean isLaunched; private String logURL; private ContainerStatus finishedStatus; @@ -164,6 +166,8 @@ public RMContainerImpl(Container container, this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); + this.isLaunched = false; + this.logURL = null; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -363,9 +367,11 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { public void transition(RMContainerImpl container, RMContainerEvent event) { // The logs of running containers should be found on NM webUI // The logs should be accessible after the container is launched - container.logURL = WebAppUtils.getLogUrl(container.container - .getNodeHttpAddress(), container.getAllocatedNode().toString(), - container.containerId, container.user); + container.isLaunched = true; + String containerId = ConverterUtils.toString(container.containerId); + container.logURL = + WebAppUtils.getLogURL("//" + container.container.getNodeHttpAddress(), + "node", "containerlogs", containerId, null, container.user); // Unregister from containerAllocationExpirer. container.containerAllocationExpirer.unregister(container .getContainerId()); @@ -380,8 +386,15 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { container.finishTime = System.currentTimeMillis(); container.finishedStatus = finishedEvent.getRemoteContainerStatus(); - // TODO: when AHS webUI is ready, logURL should be updated to point to - // the web page that will show the aggregated logs + // when AHS webUI is ready, logURL should be updated to point to + // the web page that will show the aggregated logs, if the container + // has been launched before + if (container.isLaunched) { + String containerId = ConverterUtils.toString(container.containerId); + container.logURL = WebAppUtils.getLogURL( + null, container.getAllocatedNode().toString(), + null, containerId, containerId, container.user); + } // Inform AppAttempt container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 35f21db..aaf1b74 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import static org.apache.hadoop.yarn.util.StringHelper.join; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -26,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @XmlRootElement(name = "appAttempt") @XmlAccessorType(XmlAccessType.FIELD) @@ -55,10 +54,10 @@ public AppAttemptInfo(RMAppAttempt attempt, String user) { this.containerId = masterContainer.getId().toString(); this.nodeHttpAddress = masterContainer.getNodeHttpAddress(); this.nodeId = masterContainer.getNodeId().toString(); - this.logsLink = join("//", - masterContainer.getNodeHttpAddress(), - "/node", "/containerlogs/", - ConverterUtils.toString(masterContainer.getId()), "/", user); + this.logsLink = + WebAppUtils.getLogURL("//" + masterContainer.getNodeHttpAddress(), + "node", "containerlogs", + ConverterUtils.toString(masterContainer.getId()), null, user); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 6cb6114..23e0e82 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import static org.apache.hadoop.yarn.util.StringHelper.join; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; -import com.google.common.base.Joiner; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; @@ -38,6 +35,8 @@ import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import com.google.common.base.Joiner; + @XmlRootElement(name = "app") @XmlAccessorType(XmlAccessType.FIELD) public class AppInfo { @@ -131,12 +130,11 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) { Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null) { this.amContainerLogsExist = true; - String url = join(schemePrefix, - masterContainer.getNodeHttpAddress(), - "/node", "/containerlogs/", - ConverterUtils.toString(masterContainer.getId()), - "/", app.getUser()); - this.amContainerLogs = url; + this.amContainerLogs = WebAppUtils.getLogURL( + schemePrefix + masterContainer.getNodeHttpAddress(), + "node", "containerlogs", + ConverterUtils.toString(masterContainer.getId()), null, + app.getUser()); this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index f75f9b0..7e35fd0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -104,8 +104,7 @@ public void testReleaseWhileRunning() { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals( - "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. @@ -137,6 +136,9 @@ public void testReleaseWhileRunning() { .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); + assertEquals( + "host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + rmContainer.getLogURL()); } @Test @@ -191,8 +193,7 @@ public void testExpireWhileRunning() { RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); - assertEquals( - "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", + assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions.