From a5cf51cdd03cd3aa5571bbff94006edcdef7eb14 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 9 Jan 2019 18:05:19 +0530 Subject: [PATCH] YARN-9080 --- .../timeline/EntityGroupFSTimelineStore.java | 58 ++++++++++++++++++++++ .../timeline/TestEntityGroupFSTimelineStore.java | 13 +++++ 2 files changed, 71 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 80baf89..30e4fac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.service.CompositeService; @@ -514,6 +515,62 @@ private static boolean shouldCleanAppLogDir(Path appLogPath, long now, return true; } + + // Cleans the Old Bucket Directories based upon the configured retention + void cleanBuckets(Path dirpath, FileSystem fs, long retainMillis) + throws IOException { + String bucket1Regex = "\\d{4}"; + String bucket2Regex = "\\d{3}"; + long now = Time.now(); + + PathFilter bucketDirFilter = new PathFilter() { + public boolean accept(Path clustertsPath) { + boolean validDir = false; + try { + FileStatus clustertsDir = fs.getFileStatus(clustertsPath); + if(clustertsDir.isDirectory()) { + for(FileStatus bucket1 : fs.listStatus(clustertsPath)) { + Path bucket1Path = bucket1.getPath(); + if(bucket1.isDirectory() && bucket1Path.getName(). + matches(bucket1Regex)) { + for(FileStatus bucket2 : fs.listStatus(bucket1Path)) { + Path bucket2Path = bucket2.getPath(); + if(bucket2.isDirectory() && bucket2Path.getName(). + matches(bucket2Regex)) { + validDir = true; + if((fs.listStatus(bucket2Path).length != 0) || (now + - bucket2.getModificationTime() <= logRetainMillis)) { + LOG.debug("{} not cleaned due to {}", clustertsPath, + bucket2Path); + return false; + } + } + } + } + } + } + } catch(IOException e) { + LOG.error("Unable to clean log dir " + clustertsPath, e); + validDir = false; + } + return validDir; + } + }; + + FileStatus[] oldDirs = fs.listStatus(doneRootPath, bucketDirFilter); + for(FileStatus clusterTimestampDir : oldDirs) { + Path clusterTimestampPath = clusterTimestampDir.getPath(); + try { + LOG.info("Deleting {}", clusterTimestampPath); + if (!fs.delete(clusterTimestampPath, true)) { + LOG.error("Unable to remove " + clusterTimestampPath); + } + } catch (IOException e) { + LOG.error("Unable to remove " + clusterTimestampPath, e); + } + } + } + // converts the String to an ApplicationId or null if conversion failed private static ApplicationId parseApplicationId(String appIdStr) { ApplicationId appId = null; @@ -908,6 +965,7 @@ public void run() { LOG.debug("Cleaner starting"); long startTime = Time.monotonicNow(); try { + cleanBuckets(doneRootPath, fs, logRetainMillis); cleanLogs(doneRootPath, fs, logRetainMillis); } catch (Exception e) { Throwable t = extract(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 61da3c8..b9e7cf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -332,6 +332,19 @@ public void testCleanLogs() throws Exception { } @Test + public void testCleanBuckets() throws Exception { + Path clusterTimestampDir = new Path(testDoneDirPath, + Long.toString(mainTestAppId.getClusterTimestamp())); + Path doneAppHomeDir = new Path(new Path( + clusterTimestampDir, "0000"), "000"); + fs.mkdirs(doneAppHomeDir); + Thread.sleep(2000); + store.cleanBuckets(testDoneDirPath, fs, 1000); + // ClusterTimestampDir and Bucker dirs should be cleaned up + assertFalse(fs.exists(clusterTimestampDir)); + } + + @Test public void testPluginRead() throws Exception { // Verify precondition assertEquals(EntityGroupPlugInForTest.class.getName(), -- 2.7.4 (Apple Git-66)