diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 008edf5e57e..3c7847e8b89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.v2.hs.webapp; import java.io.IOException; +import java.util.Set; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -69,6 +70,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.logaggregation.UserLogsRequest; import org.apache.hadoop.yarn.server.webapp.WrappedLogMetaRequest; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.LogServlet; @@ -441,6 +443,29 @@ public Response getRemoteLogDirPath(@Context HttpServletRequest req, return logServlet.getRemoteLogDirPath(user, appIdStr); } + @GET + @Path("/userlogs") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @InterfaceAudience.Public + @InterfaceStability.Unstable + public Response getAggregatedLogsMeta(@Context HttpServletRequest hsr, + @QueryParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String fileName, + @QueryParam(YarnWebServiceParams.FILESIZE) Set fileSize, + @QueryParam(YarnWebServiceParams.MODIFICATION_TIME) Set modificationTime, + @QueryParam(YarnWebServiceParams.APP_ID) String appIdStr, + @QueryParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, + @QueryParam(YarnWebServiceParams.NM_ID) String nmId) throws IOException { + init(); + UserLogsRequest logsRequest = new UserLogsRequest(); + logsRequest.setAppId(appIdStr); + logsRequest.setFileName(fileName); + logsRequest.setContainerId(containerIdStr); + logsRequest.setFileSize(fileSize); + logsRequest.setModificationTime(modificationTime); + logsRequest.setNodeId(nmId); + return logServlet.getContainerLogsInfo(hsr, logsRequest); + } + @GET @Path("/aggregatedlogs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationMetaCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationMetaCollector.java new file mode 100644 index 00000000000..d697fd45e50 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationMetaCollector.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.HarFs; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Facilitates a complex query of aggregated log file metadata with + * the help of file controllers. + */ +public class LogAggregationMetaCollector { + + private static final Logger LOG = LoggerFactory.getLogger( + LogAggregationMetaCollector.class); + + private UserLogsRequest logsRequest; + private Configuration conf; + + public LogAggregationMetaCollector( + UserLogsRequest logsRequest, Configuration conf) { + this.logsRequest = logsRequest; + this.conf = conf; + } + + /** + * Collects all log file metadata based on the complex query defined in + * {@code UserLogsRequest}. + * @param fileController log aggregation file format controller + * @return collection of log file metadata grouped by containers + * @throws IOException + */ + public List collect( + LogAggregationFileController fileController) throws IOException { + List containersLogMeta = new ArrayList<>(); + RemoteIterator appDirs = LogAggregationUtils. + getUserRemoteLogDir(conf, logsRequest.getUser(), + fileController.getRemoteRootLogDir(), + fileController.getRemoteRootLogDirSuffix()); + + while (appDirs.hasNext()) { + FileStatus currentAppDir = appDirs.next(); + if (logsRequest.getAppId() == null || + logsRequest.getAppId().equals(currentAppDir.getPath().getName())) { + ApplicationId appId = ApplicationId.fromString( + currentAppDir.getPath().getName()); + RemoteIterator nodeFiles = LogAggregationUtils + .getRemoteFiles(conf, currentAppDir.getPath()); + + while (nodeFiles.hasNext()) { + FileStatus currentNodeFile = nodeFiles.next(); + + if (currentNodeFile.getPath().getName().equals( + logsRequest.getAppId() + ".har")) { + Path p = new Path("har:///" + + currentNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } + + try { + Map> metaFiles = fileController + .getLogMetaFilesOfNode(logsRequest, currentNodeFile, appId); + if (metaFiles == null) { + continue; + } + containersLogMeta.addAll(createContainerLogMetas( + currentNodeFile.getPath().getName(), metaFiles)); + } catch (IOException ioe) { + LOG.warn("Can not get log meta from the log file:" + + currentNodeFile.getPath() + "\n" + ioe.getMessage()); + } + + } + } + + } + return containersLogMeta; + } + + private List createContainerLogMetas( + String nodeId, Map> metaFiles) { + List containerLogMetas = new ArrayList<>(); + for (Map.Entry> containerLogs + : metaFiles.entrySet()) { + ContainerLogMeta containerLogMeta = new ContainerLogMeta( + nodeId, containerLogs.getKey()); + for (ContainerLogFileInfo file : containerLogs.getValue()) { + boolean isFileNameMatches = logsRequest.getFileName() + .compare(file.getFileName()); + boolean fileSizeComparison = logsRequest.getFileSize() + .compare(file.getFileSize()); + boolean modificationTimeComparison = logsRequest.getModificationTime() + .compare(file.getLastModifiedTime()); + + if (!isFileNameMatches || !fileSizeComparison || + !modificationTimeComparison) { + continue; + } + containerLogMeta.getContainerLogMeta().add(file); + } + if (!containerLogMeta.getContainerLogMeta().isEmpty()) { + containerLogMetas.add(containerLogMeta); + } + } + return containerLogMetas; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java index b51be9af14d..9af75086fe9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java @@ -29,10 +29,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import com.google.common.annotations.VisibleForTesting; + +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.NoSuchElementException; @Private public class LogAggregationUtils { @@ -295,18 +298,7 @@ public static String getNodeString(String nodeId) { // Return both new and old node files combined RemoteIterator curDir = nodeFilesCur; RemoteIterator prevDir = nodeFilesPrev; - RemoteIterator nodeFilesCombined = new - RemoteIterator() { - @Override - public boolean hasNext() throws IOException { - return prevDir.hasNext() || curDir.hasNext(); - } - - @Override - public FileStatus next() throws IOException { - return prevDir.hasNext() ? prevDir.next() : curDir.next(); - } - }; + RemoteIterator nodeFilesCombined = combineIterators(prevDir, curDir); return nodeFilesCombined; } } @@ -368,4 +360,92 @@ public FileStatus next() throws IOException { return nodeFiles; } + public static RemoteIterator getRemoteFiles( + Configuration conf, Path appPath) throws IOException { + + Path qualifiedLogDir = + FileContext.getFileContext(conf).makeQualified(appPath); + return FileContext.getFileContext( + qualifiedLogDir.toUri(), conf).listStatus(appPath); + } + + public static RemoteIterator getUserRemoteLogDir( + Configuration conf, String user, Path remoteRootLogDir, + String remoteRootLogDirSuffix) throws IOException { + Path userPath = LogAggregationUtils.getRemoteLogSuffixedDir( + remoteRootLogDir, user, remoteRootLogDirSuffix); + final RemoteIterator userRootDirFiles = + getRemoteFiles(conf, userPath); + + RemoteIterator newDirs = new RemoteIterator() { + private RemoteIterator currentBucketDir = + LogAggregationUtils.getSubDir(conf, userRootDirFiles); + @Override + public boolean hasNext() throws IOException { + return currentBucketDir != null && currentBucketDir.hasNext() || + userRootDirFiles.hasNext(); + } + + @Override + public FileStatus next() throws IOException { + FileStatus next = null; + while (next == null) { + if (currentBucketDir != null && currentBucketDir.hasNext()) { + next = currentBucketDir.next(); + } else if (userRootDirFiles.hasNext()) { + currentBucketDir = LogAggregationUtils.getSubDir( + conf, userRootDirFiles); + } else { + throw new NoSuchElementException(); + } + } + return next; + } + }; + + RemoteIterator allDir = newDirs; + if (LogAggregationUtils.isOlderPathEnabled(conf)) { + try { + Path oldPath = LogAggregationUtils.getOlderRemoteLogSuffixedDir( + remoteRootLogDir, user, remoteRootLogDirSuffix); + final RemoteIterator oldUserRootDirFiles = + getRemoteFiles(conf, oldPath); + allDir = combineIterators(oldUserRootDirFiles, newDirs); + } catch (FileNotFoundException e) { + return newDirs; + } + } + + return allDir; + } + + private static RemoteIterator getSubDir( + Configuration conf, RemoteIterator rootDir) throws IOException { + if (rootDir.hasNext()) { + Path userPath = rootDir.next().getPath(); + Path qualifiedLogDir = + FileContext.getFileContext(conf).makeQualified(userPath); + return FileContext.getFileContext( + qualifiedLogDir.toUri(), conf).listStatus(userPath); + } else { + return null; + } + } + + private static RemoteIterator combineIterators( + RemoteIterator first, RemoteIterator second) { + return new RemoteIterator() { + @Override + public boolean hasNext() throws IOException { + return first.hasNext() || second.hasNext(); + } + + @Override + public FileStatus next() throws IOException { + return first.hasNext() ? first.next() : second.next(); + } + }; + + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/UserLogsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/UserLogsRequest.java new file mode 100644 index 00000000000..3bdbf33c9bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/UserLogsRequest.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation; + +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class UserLogsRequest { + private String user; + private String appId; + private String containerId; + private MatchExpression nodeId; + private MatchExpression fileName; + private ComparisonCollection fileSize; + private ComparisonCollection modificationTime; + + public static class ComparisonCollection { + private List comparisonExpressions; + + public ComparisonCollection(Set expressions) { + this.comparisonExpressions = expressions.stream() + .map(ComparisonExpression::new).collect(Collectors.toList()); + } + + public boolean compare(Long value) { + return compare(value, true); + } + + public boolean compare(String value) { + return compare(Long.valueOf(value), true); + } + + public boolean compare(Long value, boolean defaultValue) { + if (comparisonExpressions.isEmpty()) { + return defaultValue; + } + + return comparisonExpressions.stream() + .allMatch(expr -> expr.compare(value)); + } + + } + + public static class ComparisonExpression { + private static final String GREATER_SIGN = ">"; + private static final String LESSER_SIGN = "<"; + + private String expression; + private Predicate comparisonFn; + private Long convertedValue; + + public ComparisonExpression(String expression) { + if (expression == null) { + return; + } + + if (expression.startsWith(GREATER_SIGN)) { + convertedValue = Long.parseLong(expression.substring(1)); + comparisonFn = a -> a > convertedValue; + } else if (expression.startsWith(LESSER_SIGN)) { + convertedValue = Long.parseLong(expression.substring(1)); + comparisonFn = a -> a < convertedValue; + } else { + convertedValue = Long.parseLong(expression); + comparisonFn = a -> a.equals(convertedValue); + } + + this.expression = expression; + } + + public boolean compare(String value) { + return compare(Long.valueOf(value), true); + } + + public boolean compare(Long value) { + return compare(value, true); + } + + public boolean compare(Long value, boolean defaultValue) { + if (expression == null) { + return defaultValue; + } else { + return comparisonFn.test(value); + } + } + + @Override + public String toString() { + return convertedValue != null ? String.valueOf(convertedValue) : ""; + } + } + + public static class MatchExpression { + private Pattern expression; + + public MatchExpression(String expression) { + this.expression = expression != null ? Pattern.compile(expression) : null; + } + + public boolean compare(String value) { + return expression == null || expression.matcher(value).matches(); + } + + @Override + public String toString() { + return expression != null ? expression.pattern() : ""; + } + } + + public String getUser() { + return user; + } + + public String getAppId() { + return appId; + } + + public String getContainerId() { + return containerId; + } + + public MatchExpression getNodeId() { + return nodeId; + } + + public MatchExpression getFileName() { + return fileName; + } + + public void setUser(String user) { + this.user = user; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public void setNodeId(String nodeId) { + this.nodeId = new MatchExpression(nodeId); + } + + public void setFileName(String fileName) { + this.fileName = new MatchExpression(fileName); + } + + public ComparisonCollection getFileSize() { + return fileSize; + } + + public void setFileSize(Set fileSize) { + this.fileSize = new ComparisonCollection(fileSize); + } + + public ComparisonCollection getModificationTime() { + return modificationTime; + } + + public void setModificationTime(Set modificationTime) { + this.modificationTime = new ComparisonCollection(modificationTime); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 9c609beb59a..78b597da33a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -53,7 +53,9 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; +import org.apache.hadoop.yarn.logaggregation.UserLogsRequest; import org.apache.hadoop.yarn.webapp.View.ViewContext; import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block; import org.slf4j.Logger; @@ -224,6 +226,23 @@ public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest, public abstract List readAggregatedLogsMeta( ContainerLogsRequest logRequest) throws IOException; + /** + * Returns log file metadata for a node grouped by containers. + * + * @param logRequest complex query information holder + * @param currentNodeFile file status of a node in an application directory + * @param appId id of the application, which is the same as in node path + * @return log file metadata + * @throws IOException + */ + public Map> getLogMetaFilesOfNode( + UserLogsRequest logRequest, FileStatus currentNodeFile, + ApplicationId appId) throws IOException { + LOG.info("User aggregated complex log queries " + + "are not implemented for this file controller"); + return Collections.emptyMap(); + } + /** * Render Aggregated Logs block. * @param html the html diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java index 20c112b85bd..c2dd553525f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java @@ -27,6 +27,7 @@ import java.io.OutputStream; import java.io.Serializable; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.PrivilegedExceptionAction; @@ -74,12 +75,14 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; +import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue; +import org.apache.hadoop.yarn.logaggregation.UserLogsRequest; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.util.Clock; @@ -610,6 +613,49 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, return findLogs; } + @Override + public Map> getLogMetaFilesOfNode( + UserLogsRequest logRequest, FileStatus currentNodeFile, + ApplicationId appId) throws IOException { + Path nodePath = currentNodeFile.getPath(); + if (nodePath.getName().endsWith(CHECK_SUM_FILE_SUFFIX)) { + return null; + } + Map> logMetaFiles = new HashMap<>(); + + Long checkSumIndex = parseChecksum(currentNodeFile); + long endIndex = -1; + if (checkSumIndex != null) { + endIndex = checkSumIndex; + } + IndexedLogsMeta current = loadIndexedLogsMeta( + currentNodeFile.getPath(), endIndex, appId); + if (current != null) { + for (IndexedPerAggregationLogMeta logMeta : + current.getLogMetas()) { + for (Entry> log : logMeta + .getLogMetas().entrySet()) { + String currentContainerId = log.getKey(); + if (!(logRequest.getContainerId() == null || + logRequest.getContainerId().equals(currentContainerId))) { + continue; + } + logMetaFiles.put(currentContainerId, new ArrayList<>()); + for (IndexedFileLogMeta aMeta : log.getValue()) { + ContainerLogFileInfo file = new ContainerLogFileInfo(); + file.setFileName(aMeta.getFileName()); + file.setFileSize(Long.toString(aMeta.getFileSize())); + file.setLastModifiedTime( + Long.toString(aMeta.getLastModifiedTime())); + logMetaFiles.get(currentContainerId).add(file); + } + } + } + } + + return logMetaFiles; + } + @Override public List readAggregatedLogsMeta( ContainerLogsRequest logRequest) throws IOException { @@ -743,6 +789,40 @@ public int compare(ContainerLogMeta o1, ContainerLogMeta o2) { return checkSumFiles; } + public Long parseChecksum(FileStatus file) { + if (!file.getPath().getName().endsWith(CHECK_SUM_FILE_SUFFIX)) { + return null; + } + + FSDataInputStream checksumFileInputStream = null; + try { + FileContext fileContext = FileContext + .getFileContext(file.getPath().toUri(), conf); + String nodeName = null; + long index = 0L; + checksumFileInputStream = fileContext.open(file.getPath()); + int nameLength = checksumFileInputStream.readInt(); + byte[] b = new byte[nameLength]; + int actualLength = checksumFileInputStream.read(b); + if (actualLength == nameLength) { + nodeName = new String(b, StandardCharsets.UTF_8); + index = checksumFileInputStream.readLong(); + } else { + return null; + } + if (nodeName != null && !nodeName.isEmpty()) { + return index; + } + } catch (IOException ex) { + LOG.warn(ex.getMessage()); + return null; + } finally { + IOUtils.cleanupWithLogger(LOG, checksumFileInputStream); + } + + return null; + } + @Private public List getNodeLogFileToRead( List nodeFiles, String nodeId, ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index 2355d306403..6f3ff6ecfed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -25,10 +25,13 @@ import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; +import org.apache.hadoop.yarn.logaggregation.UserLogsRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.math3.util.Pair; @@ -258,6 +261,58 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, return findLogs; } + @Override + public Map> getLogMetaFilesOfNode( + UserLogsRequest logRequest, FileStatus currentNodeFile, + ApplicationId appId) throws IOException { + Map> logMetaFiles = new HashMap<>(); + Path nodePath = currentNodeFile.getPath(); + + LogReader reader = + new LogReader(conf, + nodePath); + try { + DataInputStream valueStream; + LogKey key = new LogKey(); + valueStream = reader.next(key); + while (valueStream != null) { + if (logRequest.getContainerId() == null || + logRequest.getContainerId().equals(key.toString())) { + logMetaFiles.put(key.toString(), new ArrayList<>()); + fillMetaFiles(currentNodeFile, valueStream, + logMetaFiles.get(key.toString())); + } + // Next container + key = new LogKey(); + valueStream = reader.next(key); + } + } finally { + reader.close(); + } + return logMetaFiles; + } + + private void fillMetaFiles( + FileStatus currentNodeFile, DataInputStream valueStream, + List logMetaFiles) + throws IOException { + while (true) { + try { + Pair logMeta = + LogReader.readContainerMetaDataAndSkipData( + valueStream); + ContainerLogFileInfo logMetaFile = new ContainerLogFileInfo(); + logMetaFile.setLastModifiedTime( + Long.toString(currentNodeFile.getModificationTime())); + logMetaFile.setFileName(logMeta.getFirst()); + logMetaFile.setFileSize(logMeta.getSecond()); + logMetaFiles.add(logMetaFile); + } catch (EOFException eof) { + break; + } + } + } + @Override public List readAggregatedLogsMeta( ContainerLogsRequest logRequest) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index b4f9a1f8982..7b7a76d880f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -31,8 +31,10 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta; -import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; +import org.apache.hadoop.yarn.logaggregation.LogAggregationMetaCollector; +import org.apache.hadoop.yarn.logaggregation.UserLogsRequest; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry; @@ -264,6 +266,33 @@ public Response getLogsInfo(HttpServletRequest hsr, String appIdStr, redirectedFromNode, null, manualRedirection); } + public Response getContainerLogsInfo( + HttpServletRequest req, UserLogsRequest logsRequest) throws IOException { + List logs = new ArrayList<>(); + LogAggregationMetaCollector collector = new LogAggregationMetaCollector( + logsRequest, getConf()); + if (logsRequest.getUser() == null) { + logsRequest.setUser(UserGroupInformation.getCurrentUser().getUserName()); + } + + for (LogAggregationFileController fc : getOrCreateFactory() + .getConfiguredLogAggregationFileControllerList()) { + logs.addAll(collector.collect(fc)); + } + + List containersLogsInfo = convertToContainerLogsInfo( + logs, false); + GenericEntity> meta = + new GenericEntity>(containersLogsInfo) { + }; + Response.ResponseBuilder response = Response.ok(meta); + // Sending the X-Content-Type-Options response header with the value + // nosniff will prevent Internet Explorer from MIME-sniffing a response + // away from the declared content-type. + response.header("X-Content-Type-Options", "nosniff"); + return response.build(); + } + /** * Returns information about the logs for a specific container. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 3aade3faafc..84697a389b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -40,4 +40,6 @@ String CLUSTER_ID = "clusterid"; String MANUAL_REDIRECTION = "manual_redirection"; String REMOTE_USER = "user"; + String FILESIZE = "file_size"; + String MODIFICATION_TIME = "modification_time"; }