diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index d28c77c..50d8d36 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -340,22 +340,20 @@ private boolean fetchAllLogFiles(String[] logFiles) { .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress); ClientResponse response = webResource.path("ws").path("v1").path("node").path("containers") - .path(containerIdStr).accept(MediaType.APPLICATION_XML) + .path(containerIdStr).path("logs") + .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); - if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) { + if (response.getClientResponseStatus().equals( + ClientResponse.Status.OK)) { try { - String xml = response.getEntity(String.class); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(xml)); - Document dom = db.parse(is); - NodeList elements = dom.getElementsByTagName("containerLogFiles"); - for (int i = 0; i < elements.getLength(); i++) { - logFiles.add(elements.item(i).getTextContent()); + JSONObject json = + response.getEntity(JSONObject.class); + JSONArray array = json.getJSONArray("containerLogMeta"); + for (int i = 0; i < array.length(); i++) { + logFiles.add(array.getJSONObject(i).getString("fileName")); } } catch (Exception e) { - System.err.println("Unable to parse xml from webservice. Error:"); + System.err.println("Unable to parse json from webservice. Error:"); System.err.println(e.getMessage()); throw new IOException(e); } @@ -409,7 +407,8 @@ public void printContainerLogsFromRunningApplication(Configuration conf, + nodeHttpAddress); ClientResponse response = webResource.path("ws").path("v1").path("node") - .path("containerlogs").path(containerIdStr).path(logFile) + .path("containers").path(containerIdStr).path("logs") + .path(logFile) .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); out.println(response.getEntity(String.class)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 692b172..51d43ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -76,7 +76,7 @@ public class AHSWebServices extends WebServices { private static final String NM_DOWNLOAD_URI_STR = - "/ws/v1/node/containerlogs"; + "/ws/v1/node/containers"; private static final Joiner JOINER = Joiner.on(""); private static final Joiner DOT_JOINER = Joiner.on(". "); private final Configuration conf; @@ -256,7 +256,7 @@ public Response getLogs(@Context HttpServletRequest req, String nodeId = containerInfo.getNodeId(); if (isRunningState(appInfo.getAppState())) { String nodeHttpAddress = containerInfo.getNodeHttpAddress(); - String uri = "/" + containerId.toString() + "/" + filename; + String uri = "/" + containerId.toString() + "/logs/" + filename; String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri); String query = req.getQueryString(); if (query != null && !query.isEmpty()) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 71b0275..8721a1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -715,8 +715,9 @@ public void testContainerLogsForRunningApps() throws Exception { String redirectURL = getRedirectURL(requestURI.toString()); assertTrue(redirectURL != null); assertTrue(redirectURL.contains("test:1234")); - assertTrue(redirectURL.contains("ws/v1/node/containerlogs")); + assertTrue(redirectURL.contains("ws/v1/node/containers")); assertTrue(redirectURL.contains(containerId1.toString())); + assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + user)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 3a30392..b61c845 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerLogsMeta; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -194,7 +195,67 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context .toString(), webapp.name(), hsr.getRemoteUser()); } - + + /** + * Returns log file's name as well as current file size for a container. + * + * @param containerIdStr + * The container ID + * @return + * The log file's name and current file size + */ + @GET + @Path("/containers/{containerid}/logs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public ContainerLogsMeta getContainerLogsMeta(@javax.ws.rs.core.Context + HttpServletRequest hsr, + @PathParam("containerid") String containerIdStr) { + ContainerId containerId = null; + init(); + try { + containerId = ContainerId.fromString(containerIdStr); + } catch (Exception e) { + throw new BadRequestException("invalid container id, " + containerIdStr); + } + try { + return new ContainerLogsMeta(this.nmContext, containerId, + hsr.getRemoteUser()); + } catch (YarnException ex) { + throw new WebApplicationException(ex); + } + } + + /** + * Returns the contents of a container's log file in plain text. + * + * Only works for containers that are still in the NodeManager's memory, so + * logs are no longer available after the corresponding application is no + * longer running. + * + * @param containerIdStr + * The container ID + * @param filename + * The name of the log file + * @param format + * The content type + * @param size + * the size of the log file + * @return + * The contents of the container's log file + */ + @GET + @Path("/containers/{containerid}/logs/{filename}") + @Produces({ MediaType.TEXT_PLAIN }) + @Public + @Unstable + public Response getContainerLog( + @PathParam("containerid") String containerIdStr, + @PathParam("filename") String filename, + @QueryParam("format") String format, + @QueryParam("size") String size) { + return getLogs(containerIdStr, filename, format, size); + } + /** * Returns the contents of a container's log file in plain text. * diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsMeta.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsMeta.java new file mode 100644 index 0000000..f730dbd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerLogsMeta.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp.dao; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils; + +@XmlRootElement(name = "containerLogsMeta") +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLogsMeta { + + @XmlElement(name = "containerLogMeta") + protected List containerLogsMeta; + + //JAXB needs this + public ContainerLogsMeta() {} + + public ContainerLogsMeta(final Context nmContext, + final ContainerId containerId, String remoteUser) + throws YarnException { + this.containerLogsMeta = getContainerLogsMeta( + containerId, remoteUser, nmContext); + } + + public List getContainerLogMeta() { + return this.containerLogsMeta; + } + + private List getContainerLogsMeta(ContainerId id, + String remoteUser, Context nmContext) throws YarnException { + List logFiles = new ArrayList(); + List logDirs = ContainerLogsUtils.getContainerLogDirs( + id, remoteUser, nmContext); + for (File containerLogsDir : logDirs) { + File[] logs = containerLogsDir.listFiles(); + if (logs != null) { + for (File log : logs) { + if (log.isFile()) { + ContainerLogMeta logMeta = new ContainerLogMeta( + log.getName(), log.length()); + logFiles.add(logMeta); + } + } + } + } + return logFiles; + } + + private static class ContainerLogMeta { + private String fileName; + private long fileSize; + + //JAXB needs this + public ContainerLogMeta() {} + + public ContainerLogMeta(String fileName, long fileSize) { + this.setFileName(fileName); + this.setFileSize(fileSize); + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public long getFileSize() { + return fileSize; + } + + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index a4305da..99dff79 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; @@ -313,12 +314,13 @@ public void testSingleNodesXML() throws JSONException, Exception { } @Test - public void testContainerLogs() throws IOException { + public void testContainerLogs() throws IOException, JSONException { WebResource r = resource(); final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0); final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0) .toString(); - final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); + final ApplicationAttemptId appAttemptId = containerId + .getApplicationAttemptId(); final ApplicationId appId = appAttemptId.getApplicationId(); final String appIdStr = appId.toString(); final String filename = "logfile1"; @@ -421,7 +423,22 @@ public void testContainerLogs() throws IOException { Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus()); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Cannot find this log on the local disk.")); - + + // Get container log files' name + response = r.path("ws").path("v1").path("node") + .path("containers").path(containerIdStr) + .path("logs").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(200, response.getStatus()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals(json.getJSONObject("containerLogMeta") + .getString("fileName"), filename); + + response = r.path("ws").path("v1").path("node") + .path("containers").path(containerIdStr).path("logs").path(filename) + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + assertEquals(logMessage, response.getEntity(String.class)); + // After container is completed, it is removed from nmContext nmContext.getContainers().remove(containerId); Assert.assertNull(nmContext.getContainers().get(containerId));