diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b366855..82ab8e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -719,6 +719,16 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 1000; /** + * The maximum number of applications RM returns when + * ApplicationClientProtocol.getApplications was called. + * By default equals to {@link #DEFAULT_RM_MAX_LIMIT_GET_APPLICATIONS}. + */ + public static final String RM_MAX_LIMIT_GET_APPLICATIONS = + RM_PREFIX + "max-limit-get-applications"; + public static final long DEFAULT_RM_MAX_LIMIT_GET_APPLICATIONS = + DEFAULT_RM_MAX_COMPLETED_APPLICATIONS; + + /** * The maximum number of completed applications RM state store keeps. By * default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 645a342..457450a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -698,6 +698,13 @@ + The maximum number of applications RM returns when + ApplicationClientProtocol.getApplications was called. + yarn.resourcemanager.max-limit-get-applications + ${yarn.resourcemanager.max-completed-applications} + + + Interval at which the delayed token removal thread runs yarn.resourcemanager.delayed.delegation-token.removal-interval-ms 30000 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 929a9e7..7a6efe4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -206,6 +206,8 @@ private ReservationSystem reservationSystem; private ReservationInputValidator rValidator; + private long maxLimitGetApplications; + private static final EnumSet COMPLETED_APP_STATES = EnumSet.of( RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED, RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING); @@ -239,6 +241,9 @@ public ClientRMService(RMContext rmContext, YarnScheduler scheduler, @Override protected void serviceInit(Configuration conf) throws Exception { clientBindAddress = getBindAddress(conf); + this.maxLimitGetApplications = conf.getLong( + YarnConfiguration.RM_MAX_LIMIT_GET_APPLICATIONS, + YarnConfiguration.DEFAULT_RM_MAX_LIMIT_GET_APPLICATIONS); super.serviceInit(conf); } @@ -869,6 +874,10 @@ public GetApplicationsResponse getApplications( Set queues = request.getQueues(); Set tags = request.getApplicationTags(); long limit = request.getLimit(); + if (limit > maxLimitGetApplications) { + LOG.info("User " + callerUGI.getUserName() + " called getApplications with limit=" + limit); + limit = maxLimitGetApplications; + } LongRange start = request.getStartRange(); LongRange finish = request.getFinishRange(); ApplicationsRequestScope scope = request.getScope(); @@ -1847,4 +1856,8 @@ private RMApp verifyUserAccessForRMApp(ApplicationId applicationId, return application; } + @VisibleForTesting + protected void setMaxLimitGetApplications(long limit) { + this.maxLimitGetApplications = limit; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 7a67aa8..0f24167 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1127,6 +1127,12 @@ public void handle(Event event) {} ApplicationsRequestScope.OWN); assertEquals("Incorrect number of applications for the scope", 3, rmService.getApplications(request).getApplicationList().size()); + + rmService.setMaxLimitGetApplications(1); + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.OWN); + assertEquals("Incorrect number of applications for the scope", 1, + rmService.getApplications(request).getApplicationList().size()); } @Test(timeout=4000)