diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 80baf897376..da292051e7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -469,25 +469,26 @@ void cleanLogs(Path dirpath, FileSystem fs, long retainMillis) RemoteIterator iter = list(dirpath); while (iter.hasNext()) { FileStatus stat = iter.next(); + Path childPath = stat.getPath(); if (stat.isDirectory()) { // If current is an application log dir, decide if we need to remove it // and remove if necessary. // Otherwise, keep iterating into it. - ApplicationId appId = parseApplicationId(dirpath.getName()); + ApplicationId appId = parseApplicationId(childPath.getName()); if (appId != null) { // Application log dir - if (shouldCleanAppLogDir(dirpath, now, fs, retainMillis)) { + if (shouldCleanAppLogDir(childPath, now, fs, retainMillis)) { try { - LOG.info("Deleting {}", dirpath); - if (!fs.delete(dirpath, true)) { - LOG.error("Unable to remove " + dirpath); + LOG.info("Deleting {}", childPath); + if (!fs.delete(childPath, true)) { + LOG.error("Unable to remove " + childPath); } metrics.incrLogsDirsCleaned(); } catch (IOException e) { - LOG.error("Unable to remove " + dirpath, e); + LOG.error("Unable to remove " + childPath, e); } } } else { // Keep cleaning inside - cleanLogs(stat.getPath(), fs, retainMillis); + cleanLogs(childPath, fs, retainMillis); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 04587229b0f..2fb7bef6cb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -293,7 +293,20 @@ public void testCleanLogs() throws Exception { fs.mkdirs(attemptDirEmpty); Path dirPathEmpty = new Path(attemptDirEmpty, "empty"); fs.mkdirs(dirPathEmpty); - + // Fifth application, multiple attempts and same appId child directory + Path appDirMultiAttempts = new Path(doneAppHomeDir, appDirName + "4"); + fs.mkdirs(appDirMultiAttempts); + Path attempt1 = new Path(appDirMultiAttempts, + ApplicationAttemptId.appAttemptIdStrPrefix + + appDirName + "4" + "_1"); + Path attempt2 = new Path(appDirMultiAttempts, + ApplicationAttemptId.appAttemptIdStrPrefix + + appDirName + "4" + "_2"); + Path appDirRepeated = new Path(appDirMultiAttempts, + appDirMultiAttempts.getName()); + fs.mkdirs(attempt1); + fs.mkdirs(attempt2); + fs.mkdirs(appDirRepeated); // Should retain all logs after this run MutableCounterLong dirsCleaned = store.metrics.getLogsDirsCleaned(); long before = dirsCleaned.value(); @@ -304,7 +317,7 @@ public void testCleanLogs() throws Exception { assertTrue(fs.exists(filePathHold)); assertTrue(fs.exists(dirPathHold)); assertTrue(fs.exists(dirPathEmpty)); - + assertTrue(fs.exists(appDirMultiAttempts)); // Make sure the created dir is old enough Thread.sleep(2000); // Touch the second application @@ -323,10 +336,11 @@ public void testCleanLogs() throws Exception { assertTrue(fs.exists(dirPathHold)); assertTrue(fs.exists(doneAppHomeDir)); - // appDirClean and appDirEmpty should be cleaned up + // appDirClean, appDirEmpty and appDirMultiAttempts should be cleaned up assertFalse(fs.exists(appDirClean)); assertFalse(fs.exists(appDirEmpty)); - assertEquals(before + 2L, dirsCleaned.value()); + assertFalse(fs.exists(appDirMultiAttempts)); + assertEquals(before + 3L, dirsCleaned.value()); } @Test