diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 7a05d13..63ed5c2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1447,6 +1447,16 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; + /** + * Defines how many applications can be loaded into + * timeline service web ui. + * This will also affect the number of applications fetched from + * REST API and application history protocol. + */ + public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS = + APPLICATION_HISTORY_PREFIX + "max-applications"; + public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 1200690..c3c3d9b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -33,6 +33,7 @@ String ENTITY_STRING = "entity.string"; String APP_OWNER = "app.owner"; String APP_STATE = "app.state"; + String MAX_APP = "max-applications"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; String NODE_LABEL = "node.label"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1dd88bd..7c6fdb8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1333,6 +1333,16 @@ + + Defines how many applications can be loaded into timeline service web ui. + This will also affect the number of applications fetched from REST API and + application history protocol. + + yarn.timeline-service.generic-application-history.max-applications + 10000 + + + Store class name for timeline store. yarn.timeline-service.store-class org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index e64ca14..e37462c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -184,9 +184,11 @@ public GetApplicationReportResponse getApplicationReport( public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { + long maxApps = request.getLimit(); GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList( - history.getAllApplications().values())); + maxApps == Long.MAX_VALUE ? history.getAllApplications().values() + : history.getAllApplications(maxApps).values())); return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java index 041c31b..863a549 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java @@ -63,6 +63,22 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException, throws YarnException, IOException; /** + * This method returns the given number of Application + * {@link ApplicationReport}s + * + * @param maxApplications + * + * @return map of {@link ApplicationId} to {@link ApplicationReport}s. + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + Map + getAllApplications(long maxApplications) throws YarnException, + IOException; + + /** * Application can have multiple application attempts * {@link ApplicationAttemptReport}. This method returns the all * {@link ApplicationAttemptReport}s for the Application. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index c7cf07b..5d40daa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -113,6 +113,12 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) } @Override + public Map getAllApplications(long maxApps) + throws IOException { + return getAllApplications(); + } + + @Override public ApplicationReport getApplication(ApplicationId appId) throws IOException { return convertToApplicationReport(historyStore.getApplication(appId)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 2429a68..10d36ee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -78,6 +79,7 @@ private TimelineDataManager timelineDataManager; private ApplicationACLsManager aclsManager; private String serverHttpAddress; + private long maxLoadedApplications; public ApplicationHistoryManagerOnTimelineStore( TimelineDataManager timelineDataManager, @@ -91,6 +93,9 @@ public ApplicationHistoryManagerOnTimelineStore( protected void serviceInit(Configuration conf) throws Exception { serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf); + maxLoadedApplications = + conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS); super.serviceInit(conf); } @@ -105,7 +110,31 @@ public ApplicationReport getApplication(ApplicationId appId) throws YarnException, IOException { TimelineEntities entities = timelineDataManager.getEntities( ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, - null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class), + null, null, this.maxLoadedApplications, EnumSet.allOf(Field.class), + UserGroupInformation.getLoginUser()); + Map apps = + new LinkedHashMap(); + if (entities != null && entities.getEntities() != null) { + for (TimelineEntity entity : entities.getEntities()) { + try { + ApplicationReportExt app = + generateApplicationReport(entity, ApplicationReportField.ALL); + apps.put(app.appReport.getApplicationId(), app.appReport); + } catch (Exception e) { + LOG.error("Error on generating application report for " + + entity.getEntityId(), e); + } + } + } + return apps; + } + + @Override + public Map getAllApplications( + long maxApplications) throws YarnException, IOException { + TimelineEntities entities = timelineDataManager.getEntities( + ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, + null, null, maxApplications, EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); Map apps = new LinkedHashMap(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index d03b26d..ad03718 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -53,6 +53,7 @@ public class TestApplicationHistoryClientService { private static ApplicationHistoryClientService clientService; + private static TimelineDataManager dataManager; @BeforeClass public static void setup() throws Exception { @@ -60,7 +61,7 @@ public static void setup() throws Exception { TimelineStore store = TestApplicationHistoryManagerOnTimelineStore.createStore(2); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); - TimelineDataManager dataManager = + dataManager = new TimelineDataManager(store, aclsManager); ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); ApplicationHistoryManagerOnTimelineStore historyManager = @@ -101,8 +102,27 @@ public void testApplications() throws IOException, YarnException { clientService.getApplications(request); List appReport = response.getApplicationList(); Assert.assertNotNull(appReport); - Assert.assertEquals(appId, appReport.get(0).getApplicationId()); - Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); + Assert.assertEquals(appId, appReport.get(1).getApplicationId()); + Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); + + // Create a historyManager, and set the max_apps can be loaded + // as 1. + Configuration conf = new YarnConfiguration(); + conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1); + ApplicationHistoryManagerOnTimelineStore historyManager2 = + new ApplicationHistoryManagerOnTimelineStore(dataManager, + new ApplicationACLsManager(conf)); + historyManager2.init(conf); + historyManager2.start(); + @SuppressWarnings("resource") + ApplicationHistoryClientService clientService2 = + new ApplicationHistoryClientService(historyManager2); + response = clientService2.getApplications(request); + appReport = response.getApplicationList(); + Assert.assertNotNull(appReport); + Assert.assertTrue(appReport.size() == 1); + // Expected to get the appReport for application with appId1 + Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index fcdafc2..40b3964 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -180,8 +180,10 @@ public ApplicationReport run() throws Exception { Assert.assertEquals("test app type", app.getApplicationType()); Assert.assertEquals("user1", app.getUser()); Assert.assertEquals("test queue", app.getQueue()); - Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime()); - Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime()); + Assert.assertEquals(Integer.MAX_VALUE + 2L + + app.getApplicationId().getId(), app.getStartTime()); + Assert.assertEquals(Integer.MAX_VALUE + 3L + + +app.getApplicationId().getId(), app.getFinishTime()); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. @@ -472,12 +474,12 @@ private static TimelineEntity createApplicationTimelineEntity( entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); - tEvent.setTimestamp(Integer.MAX_VALUE + 2L); + tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId()); entity.addEvent(tEvent); tEvent = new TimelineEvent(); tEvent.setEventType( ApplicationMetricsConstants.FINISHED_EVENT_TYPE); - tEvent.setTimestamp(Integer.MAX_VALUE + 3L); + tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId()); Map eventInfo = new HashMap(); eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 5df70c4..8f12a46 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.MAX_APP; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; @@ -70,10 +71,14 @@ protected void fetchData() throws YarnException, IOException, reqAppStates.add(YarnApplicationState.valueOf(stateString.trim())); } } - callerUGI = getCallerUGI(); final GetApplicationsRequest request = GetApplicationsRequest.newInstance(reqAppStates); + String maxAppsStr = $(MAX_APP); + if (maxAppsStr != null && !maxAppsStr.isEmpty()) { + long maxApps = Long.parseLong(maxAppsStr); + request.setLimit(maxApps); + } if (callerUGI == null) { appReports = appBaseProt.getApplications(request).getApplicationList(); } else {