diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 46e39cc..a4cedd2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -212,6 +212,13 @@ public void write(DataOutputStream out, Set pendingUploadFiles) Collections.sort(fileList); for (File logFile : fileList) { + // We only aggregate top level files. + // Ignore anything inside sub-folders. + if (logFile.isDirectory()) { + LOG.warn("Log file: " + logFile.getAbsolutePath() + + " is a directory. Ignore it."); + continue; + } FileInputStream in = null; try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 0fae77d..24fdf4f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -194,6 +194,14 @@ public void testReadAcontainerLogs1() throws Exception { int numChars = 80000; + // create a sub-folder under srcFilePath + // and create file logs in this sub-folder. + // We only aggregate top level files. + // So, this log file should be ignored. + Path subDir = new Path(srcFilePath, "subDir"); + fs.mkdirs(subDir); + writeSrcFile(subDir, "logs", numChars); + // create file stderr and stdout in containerLogDir writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stdout", numChars); @@ -236,6 +244,7 @@ public void testReadAcontainerLogs1() throws Exception { + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); + Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));