From eb6da403fa96365421540a3cf3366540264da423 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 6 Mar 2019 13:42:09 +0530 Subject: [PATCH] YARN-9080 --- .../timeline/EntityGroupFSTimelineStore.java | 57 ++++++++++++++++++++++ .../timeline/TestEntityGroupFSTimelineStore.java | 13 +++++ 2 files changed, 70 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 80baf89..e2545e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.service.CompositeService; @@ -514,6 +515,61 @@ private static boolean shouldCleanAppLogDir(Path appLogPath, long now, return true; } + // Cleans the Old Bucket Directories based upon the configured retention + void cleanBuckets(Path dirpath, FileSystem fs, long retainMillis) + throws IOException { + String bucket1Regex = "\\d{4}"; + String bucket2Regex = "\\d{3}"; + long now = Time.now(); + + PathFilter bucketDirFilter = new PathFilter() { + public boolean accept(Path clustertsPath) { + boolean validDir = false; + try { + FileStatus clustertsDir = fs.getFileStatus(clustertsPath); + if (clustertsDir.isDirectory()) { + for (FileStatus bucket1 : fs.listStatus(clustertsPath)) { + Path bucket1Path = bucket1.getPath(); + if (bucket1.isDirectory() && bucket1Path.getName(). + matches(bucket1Regex)) { + for (FileStatus bucket2 : fs.listStatus(bucket1Path)) { + Path bucket2Path = bucket2.getPath(); + if (bucket2.isDirectory() && bucket2Path.getName(). + matches(bucket2Regex)) { + validDir = true; + if ((fs.listStatus(bucket2Path).length != 0) || (now + - bucket2.getModificationTime() <= retainMillis)) { + LOG.debug("{} not cleaned due to {}", clustertsPath, + bucket2Path); + return false; + } + } + } + } + } + } + } catch(IOException e) { + LOG.error("Unable to clean log dir {}", clustertsPath, e); + validDir = false; + } + return validDir; + } + }; + + FileStatus[] oldDirs = fs.listStatus(doneRootPath, bucketDirFilter); + for (FileStatus clusterTimestampDir : oldDirs) { + Path clusterTimestampPath = clusterTimestampDir.getPath(); + try { + LOG.info("Deleting {}", clusterTimestampPath); + if (!fs.delete(clusterTimestampPath, true)) { + LOG.error("Unable to remove {}", clusterTimestampPath); + } + } catch (IOException e) { + LOG.error("Unable to remove {}", clusterTimestampPath, e); + } + } + } + // converts the String to an ApplicationId or null if conversion failed private static ApplicationId parseApplicationId(String appIdStr) { ApplicationId appId = null; @@ -908,6 +964,7 @@ public void run() { LOG.debug("Cleaner starting"); long startTime = Time.monotonicNow(); try { + cleanBuckets(doneRootPath, fs, logRetainMillis); cleanLogs(doneRootPath, fs, logRetainMillis); } catch (Exception e) { Throwable t = extract(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 61da3c8..c8d07ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -332,6 +332,19 @@ public void testCleanLogs() throws Exception { } @Test + public void testCleanBuckets() throws Exception { + Path clusterTimestampDir = new Path(testDoneDirPath, + Long.toString(mainTestAppId.getClusterTimestamp())); + Path doneAppHomeDir = new Path(new Path( + clusterTimestampDir, "0000"), "000"); + fs.mkdirs(doneAppHomeDir); + Thread.sleep(2000); + store.cleanBuckets(testDoneDirPath, fs, 1000); + // ClusterTimestampDir and Bucker dirs should be cleaned up + assertFalse(fs.exists(clusterTimestampDir)); + } + + @Test public void testPluginRead() throws Exception { // Verify precondition assertEquals(EntityGroupPlugInForTest.class.getName(), -- 2.7.4 (Apple Git-66)