diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 81d63db..877d0a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -846,7 +846,7 @@ public GetApplicationsResponse getApplications( ApplicationsRequestScope scope = request.getScope(); final Map apps = rmContext.getRMApps(); - Iterator appsIter; + Iterator appsIter; // If the query filters by queues, we can avoid considering apps outside // of those queues by asking the scheduler for the apps in those queues. if (queues != null && !queues.isEmpty()) { @@ -860,7 +860,7 @@ public GetApplicationsResponse getApplications( queueAppLists.add(appsInQueue); } } - appsIter = new Iterator() { + appsIter = new Iterator() { Iterator> appListIter = queueAppLists.iterator(); Iterator schedAppsIter; @@ -872,11 +872,11 @@ public boolean hasNext() { || appListIter.hasNext(); } @Override - public RMApp next() { + public ApplicationId next() { if (schedAppsIter == null || !schedAppsIter.hasNext()) { schedAppsIter = appListIter.next().iterator(); } - return apps.get(schedAppsIter.next().getApplicationId()); + return schedAppsIter.next().getApplicationId(); } @Override public void remove() { @@ -884,13 +884,13 @@ public void remove() { } }; } else { - appsIter = apps.values().iterator(); + appsIter = rmAppManager.getRMAppsIterator(); } List reports = new ArrayList(); while (appsIter.hasNext() && reports.size() < limit) { - RMApp application = appsIter.next(); + RMApp application = apps.get(appsIter.next()); // Check if current application falls under the specified scope if (scope == ApplicationsRequestScope.OWN && !callerUGI.getUserName().equals(application.getUser())) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 49daedb..6be4abb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -19,8 +19,11 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.Iterator; import java.util.LinkedList; import java.util.Map; +import java.util.concurrent.ConcurrentSkipListSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -87,6 +90,16 @@ private Configuration conf; private YarnAuthorizationProvider authorizer; + private final ConcurrentSkipListSet copyOfApps; + + static final Comparator appsComparator = + new Comparator() { + @Override + public int compare(ApplicationId a1, ApplicationId a2) { + return a1.compareTo(a2); + } + }; + public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { @@ -106,6 +119,7 @@ public RMAppManager(RMContext context, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.copyOfApps = new ConcurrentSkipListSet<>(appsComparator); } /** @@ -278,6 +292,7 @@ protected synchronized void checkAppNumCompletedLimit() { + this.maxCompletedAppsInMemory + ", removing app " + removeId + " from memory: "); rmContext.getRMApps().remove(removeId); + copyOfApps.remove(removeId); this.applicationACLsManager.removeApplication(removeId); } } @@ -405,6 +420,7 @@ private RMAppImpl createAndPopulateNewRMApp( throw new YarnException(message); } + copyOfApps.add(applicationId); if (YarnConfiguration.timelineServiceV2Enabled(conf)) { // Start timeline collector for the submitted app application.startTimelineCollector(); @@ -506,4 +522,9 @@ public void handle(RMAppManagerEvent event) { LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!"); } } + + public Iterator getRMAppsIterator() { + // provide latest one as first + return copyOfApps.descendingIterator(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 2a7971e..880484e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -38,6 +38,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -779,9 +780,17 @@ public void handle(Event event) {} rmService.getApplications(request).getApplicationList().size()); // Check limit - request.setLimit(1L); - assertEquals("Failed to limit applications", 1, - rmService.getApplications(request).getApplicationList().size()); + request.setLimit(3); + List applicationList = + rmService.getApplications(request).getApplicationList(); + assertEquals("Failed to limit applications", 3, applicationList.size()); + + Iterator iterator = applicationList.iterator(); + for (int j = 3; j > 0; j--) { + // verify for applicationId + assertEquals("Latest Application is not retrieved", appIds[j - 1], + iterator.next().getApplicationId()); + } // Check start range request = GetApplicationsRequest.newInstance();