diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index e1e684b..5c80132 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -47,7 +47,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -61,7 +60,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; -import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java index 1566ae2..359f421 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java @@ -20,18 +20,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicInteger; /** @@ -47,15 +42,12 @@ private EntityGroupFSTimelineStore.AppLogs appLogs; private long lastRefresh; private Configuration config; - private FileSystem fs; private int refCount = 0; private static AtomicInteger activeStores = new AtomicInteger(0); - public EntityCacheItem(TimelineEntityGroupId gId, Configuration config, - FileSystem fs) { + public EntityCacheItem(TimelineEntityGroupId gId, Configuration config) { this.groupId = gId; this.config = config; - this.fs = fs; } /** @@ -97,15 +89,12 @@ public static int getActiveStores() { * other operations on the same cache item. * * @param aclManager ACL manager for the timeline storage - * @param jsonFactory JSON factory for the storage - * @param objMapper Object mapper for the storage * @param metrics Metrics to trace the status of the entity group store * @return a {@link org.apache.hadoop.yarn.server.timeline.TimelineStore} * object filled with all entities in the group. * @throws IOException */ public synchronized TimelineStore refreshCache(TimelineACLsManager aclManager, - JsonFactory jsonFactory, ObjectMapper objMapper, EntityGroupFSTimelineStoreMetrics metrics) throws IOException { if (needRefresh()) { long startTime = Time.monotonicNow(); @@ -128,30 +117,13 @@ public synchronized TimelineStore refreshCache(TimelineACLsManager aclManager, // Store is not null, the refresh is triggered by stale storage. metrics.incrCacheStaleRefreshes(); } - List removeList = new ArrayList<>(); try (TimelineDataManager tdm = new TimelineDataManager(store, aclManager)) { tdm.init(config); tdm.start(); - for (LogInfo log : appLogs.getDetailLogs()) { - LOG.debug("Try refresh logs for {}", log.getFilename()); - // Only refresh the log that matches the cache id - if (log.matchesGroupId(groupId)) { - Path appDirPath = appLogs.getAppDirPath(); - if (fs.exists(log.getPath(appDirPath))) { - LOG.debug("Refresh logs for cache id {}", groupId); - log.parseForStore(tdm, appDirPath, appLogs.isDone(), - jsonFactory, objMapper, fs); - } else { - // The log may have been removed, remove the log - removeList.add(log); - LOG.info("File {} no longer exists, removing it from log list", - log.getPath(appDirPath)); - } - } - } + // Load data from appLogs to tdm + appLogs.loadDetailLog(tdm, groupId); } - appLogs.getDetailLogs().removeAll(removeList); } updateRefreshTimeToNow(); metrics.addCacheRefreshTime(Time.monotonicNow() - startTime); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 7e6c154..9593a31 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -230,7 +230,6 @@ protected boolean removeEldestEntry( List pluginList = new LinkedList(); - Exception caught = null; ClassLoader customClassLoader = null; if (pluginClasspath != null && pluginClasspath.length() > 0) { try { @@ -775,8 +774,8 @@ private void addSummaryLog(String attemptDirName, summaryLogs.add(log); } - private void addDetailLog(String attemptDirName, String filename, - String owner) { + private synchronized void addDetailLog(String attemptDirName, + String filename, String owner) { for (LogInfo log : detailLogs) { if (log.getFilename().equals(filename) && log.getAttemptDirName().equals(attemptDirName)) { @@ -786,6 +785,30 @@ private void addDetailLog(String attemptDirName, String filename, detailLogs.add(new EntityLogInfo(attemptDirName, filename, owner)); } + synchronized void loadDetailLog(TimelineDataManager tdm, + TimelineEntityGroupId groupId) throws IOException { + List removeList = new ArrayList<>(); + for (LogInfo log : detailLogs) { + LOG.debug("Try refresh logs for {}", log.getFilename()); + // Only refresh the log that matches the cache id + if (log.matchesGroupId(groupId)) { + Path dirPath = getAppDirPath(); + if (fs.exists(log.getPath(dirPath))) { + LOG.debug("Refresh logs for cache id {}", groupId); + log.parseForStore(tdm, dirPath, isDone(), + jsonFactory, objMapper, fs); + } else { + // The log may have been removed, remove the log + removeList.add(log); + LOG.info( + "File {} no longer exists, removing it from log list", + log.getPath(dirPath)); + } + } + } + detailLogs.removeAll(removeList); + } + public synchronized void moveToDone() throws IOException { Path doneAppPath = getDoneAppPath(appId); if (!doneAppPath.equals(appDirPath)) { @@ -974,7 +997,7 @@ private TimelineStore getCachedStore(TimelineEntityGroupId groupId, cacheItem = this.cachedLogs.get(groupId); if (cacheItem == null) { LOG.debug("Set up new cache item for id {}", groupId); - cacheItem = new EntityCacheItem(groupId, getConfig(), fs); + cacheItem = new EntityCacheItem(groupId, getConfig()); AppLogs appLogs = getAndSetAppLogs(groupId.getApplicationId()); if (appLogs != null) { LOG.debug("Set applogs {} for group id {}", appLogs, groupId); @@ -994,8 +1017,7 @@ private TimelineStore getCachedStore(TimelineEntityGroupId groupId, // Add the reference by the store cacheItem.incrRefs(); cacheItems.add(cacheItem); - store = cacheItem.refreshCache(aclManager, jsonFactory, objMapper, - metrics); + store = cacheItem.refreshCache(aclManager, metrics); } else { LOG.warn("AppLogs for group id {} is null", groupId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index 95d0987..5a0822d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -354,7 +354,7 @@ public void testPluginRead() throws Exception { AppState.COMPLETED); EntityCacheItem cacheItem = new EntityCacheItem( EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), - config, fs); + config); cacheItem.setAppLogs(appLogs); store.setCachedLogs( EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), @@ -406,7 +406,7 @@ public void testMultiplePluginRead() throws Exception { AppState.COMPLETED); final EntityCacheItem cacheItem = new EntityCacheItem( EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), - config, fs); + config); cacheItem.setAppLogs(appLogs); store.setCachedLogs( @@ -442,7 +442,7 @@ public TimelineEntity call() throws Exception { AppState.COMPLETED); EntityCacheItem item = new EntityCacheItem( EntityGroupPlugInForTest.getStandardTimelineGroupId(appId), - config, fs); + config); item.setAppLogs(currAppLog); store.setCachedLogs( EntityGroupPlugInForTest.getStandardTimelineGroupId(appId),