diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7a05d13..63ed5c2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1447,6 +1447,16 @@ private static void addDeprecatedKeys() {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines how many applications can be loaded into
+ * timeline service web ui.
+ * This will also affect the number of applications fetched from
+ * REST API and application history protocol.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 1200690..c3c3d9b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String MAX_APP = "max-applications";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1dd88bd..7c6fdb8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1333,6 +1333,16 @@
+
+ Defines how many applications can be loaded into timeline service web ui.
+ This will also affect the number of applications fetched from REST API and
+ application history protocol.
+
+ yarn.timeline-service.generic-application-history.max-applications
+ 10000
+
+
+
Store class name for timeline store.
yarn.timeline-service.store-class
org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e64ca14..e37462c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -184,9 +184,11 @@ public GetApplicationReportResponse getApplicationReport(
public GetApplicationsResponse
getApplications(GetApplicationsRequest request) throws YarnException,
IOException {
+ long maxApps = request.getLimit();
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList(
- history.getAllApplications().values()));
+ maxApps == Long.MAX_VALUE ? history.getAllApplications().values()
+ : history.getAllApplications(maxApps).values()));
return response;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..e4d71fd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -63,6 +63,22 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException,
throws YarnException, IOException;
/**
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s
+ *
+ * @param maxApplications
+ *
+ * @return map of {@link ApplicationId} to {@link ApplicationReport}s.
+ * @throws YarnException
+ * @throws IOException
+ */
+ @Public
+ @Unstable
+ Map
+ getAllApplications(long maxApplications) throws YarnException,
+ IOException;
+
+ /**
* Application can have multiple application attempts
* {@link ApplicationAttemptReport}. This method returns the all
* {@link ApplicationAttemptReport}s for the Application.
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..5d40daa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -113,6 +113,12 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
}
@Override
+ public Map getAllApplications(long maxApps)
+ throws IOException {
+ return getAllApplications();
+ }
+
+ @Override
public ApplicationReport getApplication(ApplicationId appId)
throws IOException {
return convertToApplicationReport(historyStore.getApplication(appId));
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 2429a68..10d36ee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -50,6 +50,7 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -78,6 +79,7 @@
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -91,6 +93,9 @@ public ApplicationHistoryManagerOnTimelineStore(
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -105,7 +110,31 @@ public ApplicationReport getApplication(ApplicationId appId)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
+ null, null, this.maxLoadedApplications, EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ Map apps =
+ new LinkedHashMap();
+ if (entities != null && entities.getEntities() != null) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ try {
+ ApplicationReportExt app =
+ generateApplicationReport(entity, ApplicationReportField.ALL);
+ apps.put(app.appReport.getApplicationId(), app.appReport);
+ } catch (Exception e) {
+ LOG.error("Error on generating application report for " +
+ entity.getEntityId(), e);
+ }
+ }
+ }
+ return apps;
+ }
+
+ @Override
+ public Map getAllApplications(
+ long maxApplications) throws YarnException, IOException {
+ TimelineEntities entities = timelineDataManager.getEntities(
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
+ null, null, maxApplications, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map apps =
new LinkedHashMap();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index d03b26d..ad03718 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -53,6 +53,7 @@
public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -60,7 +61,7 @@ public static void setup() throws Exception {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -101,8 +102,27 @@ public void testApplications() throws IOException, YarnException {
clientService.getApplications(request);
List appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fcdafc2..dc32588 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -472,12 +472,12 @@ private static TimelineEntity createApplicationTimelineEntity(
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(System.currentTimeMillis());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(System.currentTimeMillis());
Map eventInfo = new HashMap();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 5df70c4..8f12a46 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.MAX_APP;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -70,10 +71,14 @@ protected void fetchData() throws YarnException, IOException,
reqAppStates.add(YarnApplicationState.valueOf(stateString.trim()));
}
}
-
callerUGI = getCallerUGI();
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String maxAppsStr = $(MAX_APP);
+ if (maxAppsStr != null && !maxAppsStr.isEmpty()) {
+ long maxApps = Long.parseLong(maxAppsStr);
+ request.setLimit(maxApps);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {