diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 035d61d..22219be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -213,6 +213,12 @@ public void write(DataOutputStream out, Set pendingUploadFiles) Collections.sort(fileList); for (File logFile : fileList) { + // We only aggregate top level files. + // Ignore anything inside sub-folders. + if (logFile.isDirectory()) { + LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it."); + continue; + } FileInputStream in = null; try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index fa9de74..1d607b1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -194,6 +194,14 @@ public void testReadAcontainerLogs1() throws Exception { int numChars = 80000; + // create a sub-folder under srcFilePath + // and create file logs in this sub-folder. + // We only aggregate top level files. + // So, this log file should be ignored. + Path subDir = new Path(srcFilePath, "subDir"); + fs.mkdirs(subDir); + writeSrcFile(subDir, "logs", numChars); + // create file stderr and stdout in containerLogDir writeSrcFile(srcFilePath, "stderr", numChars); writeSrcFile(srcFilePath, "stdout", numChars); @@ -238,6 +246,7 @@ public void testReadAcontainerLogs1() throws Exception { + "\nLog Contents:\n".length() + numChars; Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); + Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars)); Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));