From 4e5082846743ce7a8fcd7bfe142c568f9f9f895e Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 9 Sep 2019 00:07:48 +0530 Subject: [PATCH] YARN-9816. Ignore unexpected file in ATS Active Directory. --- .../server/timeline/EntityGroupFSTimelineStore.java | 7 ++++++- .../server/timeline/TestEntityGroupFSTimelineStore.java | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index e10eb1b..8f00b85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -376,7 +376,12 @@ int scanActiveLogs(Path dir) throws IOException { AppLogs logs = getAndSetActiveLog(appId, stat.getPath()); executor.execute(new ActiveLogParser(logs)); } else { - logsToScanCount += scanActiveLogs(stat.getPath()); + if (stat.isDirectory()) { + logsToScanCount += scanActiveLogs(stat.getPath()); + } else { + LOG.warn("Ignoring unexpected file in active directory {}", + stat.getPath()); + } } } return logsToScanCount; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 8fcc696..984e157 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -511,6 +511,23 @@ public void testGetEntityPluginRead() throws Exception { } @Test + public void testScanActiveLogsWithInvalidFile() throws Exception { + Path invalidFile = new Path(testActiveDirPath, "invalidfile"); + try { + if (!fs.exists(invalidFile)) { + fs.createNewFile(invalidFile); + } + store.scanActiveLogs(); + } catch (StackOverflowError error) { + Assert.fail("EntityLogScanner crashed with StackOverflowError"); + } finally { + if (fs.exists(invalidFile)) { + fs.delete(invalidFile, false); + } + } + } + + @Test public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception { EntityGroupFSTimelineStore store = null; ApplicationId appId = -- 2.7.4 (Apple Git-66)