diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 7a05d13..e1a156a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1447,6 +1447,11 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + /** Defines how many applications can be loaded into timeline service web ui.*/ + public static final String TIMELINE_SERVICE_WEBAPP_MAX_APPS = + TIMELINE_SERVICE_PREFIX + "webapp.max-applications"; + public static final long DEFAULT_TIMELINE_SERVICE_WEBAPP_MAX_APPS = 10000; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1dd88bd..eb53308 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1333,6 +1333,14 @@ + + Defines how many applications can be loaded into timeline service web ui. + + yarn.timeline-service.webapp.max-applications + 10000 + + + Store class name for timeline store. yarn.timeline-service.store-class org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index db00d2c..7706cb0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -74,6 +75,7 @@ private TimelineDataManager timelineDataManager; private ApplicationACLsManager aclsManager; private String serverHttpAddress; + private long maxLoadedApplications; public ApplicationHistoryManagerOnTimelineStore( TimelineDataManager timelineDataManager, @@ -87,6 +89,9 @@ public ApplicationHistoryManagerOnTimelineStore( protected void serviceInit(Configuration conf) throws Exception { serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + maxLoadedApplications = + conf.getLong(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_MAX_APPS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_MAX_APPS); super.serviceInit(conf); } @@ -101,7 +106,7 @@ public ApplicationReport getApplication(ApplicationId appId) throws YarnException, IOException { TimelineEntities entities = timelineDataManager.getEntities( ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, - null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), + null, null, this.maxLoadedApplications, EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); Map apps = new LinkedHashMap(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index d03b26d..33a6f0c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -53,6 +53,7 @@ public class TestApplicationHistoryClientService { private static ApplicationHistoryClientService clientService; + private static TimelineDataManager dataManager; @BeforeClass public static void setup() throws Exception { @@ -60,7 +61,7 @@ public static void setup() throws Exception { TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.createStore(2); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); - TimelineDataManager dataManager = + dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = @@ -103,6 +104,24 @@ public void testApplications() throws IOException, YarnException { Assert.assertNotNull(appReport); Assert.assertEquals(appId, appReport.get(0).getApplicationId()); Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); + + // Create a historyManager, and set the max_apps can be loaded + // as 1. + Configuration conf = new YarnConfiguration(); + conf.setLong(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_MAX_APPS, 1); + ApplicationHistoryManagerOnTimelineStore historyManager2 = + new ApplicationHistoryManagerOnTimelineStore(dataManager, + new ApplicationACLsManager(conf)); + historyManager2.init(conf); + historyManager2.start(); + @SuppressWarnings("resource") + ApplicationHistoryClientService clientService2 = + new ApplicationHistoryClientService(historyManager2); + response = clientService2.getApplications(request); + appReport = response.getApplicationList(); + Assert.assertNotNull(appReport); + Assert.assertTrue(appReport.size() == 1); + Assert.assertEquals(appId, appReport.get(0).getApplicationId()); } @Test