diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 825fb25..bc9766a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -21,16 +21,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.ConnectException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.NavigableSet; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -902,8 +894,26 @@ private void addDirectoryToJobListCache(Path path) throws IOException { * @return list of history directories */ protected List findTimestampedDirectories() throws IOException { - List fsList = JobHistoryUtils.localGlobber(doneDirFc, - doneDirPrefixPath, DONE_BEFORE_SERIAL_TAIL); + List fsList = new LinkedList<>(); + long maxHistoryAgeInDay = maxHistoryAge / (24 * 60 * 60 * 1000); + if (maxHistoryAgeInDay > 0) { + LOG.info("Max history age in day is " + maxHistoryAgeInDay); + List> subDirs = new LinkedList<>(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy/MM/dd"); + Calendar calendar = Calendar.getInstance(); + for (int i = 0 ; i <= maxHistoryAgeInDay + 1 ; ++i) { + calendar.add(Calendar.DATE, (i == 0) ? 0 : -1); + String dataStr = simpleDateFormat.format(calendar.getTime()); + subDirs.add(JobHistoryUtils.localGlobber(doneDirFc, + new Path(doneDirPrefixPath.toUri().toString() + "/" + dataStr) , "")); + } + for (int i = 0; i < subDirs.size(); ++i) { + fsList.addAll(subDirs.get(i)); + } + } else { + fsList = JobHistoryUtils.localGlobber(doneDirFc, + doneDirPrefixPath, DONE_BEFORE_SERIAL_TAIL); + } return fsList; }