From b93277f0a42c15b7a6e2f2b0f0acd9ff7ad0243b Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 20 Mar 2019 22:50:02 +0530 Subject: [PATCH] YARN-9080 --- .../timeline/EntityGroupFSTimelineStore.java | 78 ++++++++++++++++------ .../timeline/TestEntityGroupFSTimelineStore.java | 16 ++++- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 80baf89..c629e08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -464,32 +464,70 @@ private AppLogs getAndSetAppLogs(ApplicationId applicationId) @VisibleForTesting void cleanLogs(Path dirpath, FileSystem fs, long retainMillis) throws IOException { + String bucket1Regex = "\\d{4}"; + String bucket2Regex = "\\d{3}"; long now = Time.now(); - // Depth first search from root directory for all application log dirs - RemoteIterator iter = list(dirpath); - while (iter.hasNext()) { - FileStatus stat = iter.next(); - if (stat.isDirectory()) { - // If current is an application log dir, decide if we need to remove it - // and remove if necessary. - // Otherwise, keep iterating into it. - ApplicationId appId = parseApplicationId(dirpath.getName()); - if (appId != null) { // Application log dir - if (shouldCleanAppLogDir(dirpath, now, fs, retainMillis)) { - try { - LOG.info("Deleting {}", dirpath); - if (!fs.delete(dirpath, true)) { - LOG.error("Unable to remove " + dirpath); + + RemoteIterator clustertsIter = list(dirpath); + while (clustertsIter.hasNext()) { + FileStatus clustertsStat = clustertsIter.next(); + if (clustertsStat.isDirectory()) { + boolean toBeRemoved = true; + boolean validClusterTsDir = false; + Path clustertsPath = clustertsStat.getPath(); + RemoteIterator bucket1Iter = list(clustertsPath); + while (bucket1Iter.hasNext()) { + FileStatus bucket1Stat = bucket1Iter.next(); + Path bucket1Path = bucket1Stat.getPath(); + if (bucket1Stat.isDirectory() && + bucket1Path.getName().matches(bucket1Regex)) { + RemoteIterator bucket2Iter = list( + bucket1Stat.getPath()); + while (bucket2Iter.hasNext()) { + FileStatus bucket2Stat = bucket2Iter.next(); + Path bucket2Path = bucket2Stat.getPath(); + if (bucket2Stat.isDirectory() && + bucket2Path.getName().matches(bucket2Regex)) { + validClusterTsDir = true; + if ((fs.listStatus(bucket2Path).length != 0) || (now + - bucket2Stat.getModificationTime() <= retainMillis)) { + toBeRemoved = false; + RemoteIterator appsIter = list(bucket2Path); + while (appsIter.hasNext()) { + FileStatus appStat = appsIter.next(); + if (appStat.isDirectory()) { + Path appPath = appStat.getPath(); + ApplicationId appId = + parseApplicationId(appPath.getName()); + if (appId != null) { // Application log dir + if (shouldCleanAppLogDir(appPath, + now, fs, retainMillis)) { + deleteDir(appPath); + } + } + } + } + } } - metrics.incrLogsDirsCleaned(); - } catch (IOException e) { - LOG.error("Unable to remove " + dirpath, e); } } - } else { // Keep cleaning inside - cleanLogs(stat.getPath(), fs, retainMillis); } + if (toBeRemoved && validClusterTsDir) { + deleteDir(clustertsPath); + } + } + } + } + + void deleteDir(Path path) { + try { + LOG.info("Deleting {}", path); + if (!fs.delete(path, true)) { + LOG.error("Unable to remove {}", path); } + metrics.incrLogsDirsCleaned(); + } catch (IOException e) { + LOG.error("Unable to remove {}", path, e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 61da3c8..873cb01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -267,7 +267,8 @@ public void testCleanLogs() throws Exception { Path irrelevantDirPath = new Path(testDoneDirPath, "irrelevant"); fs.mkdirs(irrelevantDirPath); - Path doneAppHomeDir = new Path(new Path(testDoneDirPath, "0000"), "001"); + Path doneAppHomeDir = new Path(new Path(new Path(testDoneDirPath, + Long.toString(mainTestAppId.getClusterTimestamp())), "0000"), "001"); // First application, untouched after creation Path appDirClean = new Path(doneAppHomeDir, appDirName); Path attemptDirClean = new Path(appDirClean, attemptDirName); @@ -332,6 +333,19 @@ public void testCleanLogs() throws Exception { } @Test + public void testCleanBuckets() throws Exception { + Path clusterTimestampDir = new Path(testDoneDirPath, + Long.toString(mainTestAppId.getClusterTimestamp())); + Path doneAppHomeDir = new Path(new Path( + clusterTimestampDir, "0000"), "000"); + fs.mkdirs(doneAppHomeDir); + Thread.sleep(2000); + store.cleanLogs(testDoneDirPath, fs, 1000); + // ClusterTimestampDir and Bucket dirs should be cleaned up + assertFalse(fs.exists(clusterTimestampDir)); + } + + @Test public void testPluginRead() throws Exception { // Verify precondition assertEquals(EntityGroupPlugInForTest.class.getName(), -- 2.7.4 (Apple Git-66)