diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 9d2c739..8a03095 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -73,7 +73,7 @@ @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; - @Metric("# of active users") MutableGaugeInt activeApplications; + @Metric("# of active applications") MutableGaugeInt activeApplications; private final MutableGaugeInt[] runningTime; private TimeBucketMetrics runBuckets; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a09ea61..123d7fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -290,19 +290,19 @@ private synchronized void setupQueueConfigs( "maxApplicationsPerUser = " + maxApplicationsPerUser + " [= (int)(maxApplications * (userLimit / 100.0f) * " + "userLimitFactor) ]" + "\n" + - "maxActiveApplications = " + maxActiveApplications + + "maxSchedulableApplications = " + maxActiveApplications + " [= max(" + "(int)ceil((clusterResourceMemory / minimumAllocation) * " + "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," + "1) ]" + "\n" + - "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap + + "maxSchedulableAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap + " [= max(" + "(int)ceil((clusterResourceMemory / minimumAllocation) *" + "maxAMResourcePercent * absoluteCapacity)," + "1) ]" + "\n" + - "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser + + "maxSchedulableApplicationsPerUser = " + maxActiveApplicationsPerUser + " [= max(" + - "(int)(maxActiveApplications * (userLimit / 100.0f) * " + + "(int)(maxSchedulableApplications * (userLimit / 100.0f) * " + "userLimitFactor)," + "1) ]" + "\n" + "usedCapacity = " + usedCapacity + @@ -481,32 +481,56 @@ synchronized void setUserLimitFactor(int userLimitFactor) { @Override public synchronized int getNumApplications() { - return getNumPendingApplications() + getNumActiveApplications(); + return getNumNonSchedulableApplications() + getNumSchedulableApplications(); } - public synchronized int getNumPendingApplications() { + public synchronized int getNumNonSchedulableApplications() { return pendingApplications.size(); } - public synchronized int getNumActiveApplications() { + public synchronized int getNumSchedulableApplications() { return activeApplications.size(); } + public synchronized int getNumRunningApplications(){ + return getMetrics().getAppsRunning(); + } + + public synchronized int getNumPendingApplications(){ + return getMetrics().getAppsPending(); + } + @Private public synchronized int getNumApplications(String user) { return getUser(user).getTotalApplications(); } @Private - public synchronized int getNumPendingApplications(String user) { - return getUser(user).getPendingApplications(); + public synchronized int getNumNonSchedulableApplications(String user) { + return getUser(user).getNonSchedulableApplications(); } @Private - public synchronized int getNumActiveApplications(String user) { - return getUser(user).getActiveApplications(); + public synchronized int getNumSchedulableApplications(String user) { + return getUser(user).getSchedulableApplications(); } - + + public synchronized int getNumRunningApplications(String user){ + // Enable User Metrics to make this work + if(getMetrics().getUserMetrics(user) == null){ + return 0; + } + return getMetrics().getUserMetrics(user).getAppsRunning(); + } + + public synchronized int getNumPendingApplications(String user){ + // Enable User Metrics to make this work + if(getMetrics().getUserMetrics(user) == null){ + return 0; + } + return getMetrics().getUserMetrics(user).getAppsPending(); + } + public synchronized int getNumContainers() { return numContainers; } @@ -585,10 +609,12 @@ private synchronized User getUser(String userName) { */ public synchronized ArrayList getUsers() { ArrayList usersToReturn = new ArrayList(); - for (Map.Entry entry: users.entrySet()) { - usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone( - entry.getValue().consumed), entry.getValue().getActiveApplications(), - entry.getValue().getPendingApplications())); + for (Map.Entry entry : users.entrySet()) { + usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(entry + .getValue().consumed), entry.getValue().getSchedulableApplications(), + entry.getValue().getNonSchedulableApplications(), + getNumRunningApplications(entry.getKey()), + getNumPendingApplications(entry.getKey()))); } return usersToReturn; } @@ -708,13 +734,13 @@ private synchronized void activateApplications() { FiCaSchedulerApp application = i.next(); // Check queue limit - if (getNumActiveApplications() >= getMaximumActiveApplications()) { + if (getNumSchedulableApplications() >= getMaximumActiveApplications()) { break; } // Check user limit User user = getUser(application.getUser()); - if (user.getActiveApplications() < getMaximumActiveApplicationsPerUser()) { + if (user.getSchedulableApplications() < getMaximumActiveApplicationsPerUser()) { user.activateApplication(); activeApplications.add(application); i.remove(); @@ -737,10 +763,10 @@ private synchronized void addApplication(FiCaSchedulerApp application, User user LOG.info("Application added -" + " appId: " + application.getApplicationId() + " user: " + user + "," + " leaf-queue: " + getQueueName() + - " #user-pending-applications: " + user.getPendingApplications() + - " #user-active-applications: " + user.getActiveApplications() + - " #queue-pending-applications: " + getNumPendingApplications() + - " #queue-active-applications: " + getNumActiveApplications() + " #user-non-schedulable-applications: " + user.getNonSchedulableApplications() + + " #user-schedulable-applications: " + user.getSchedulableApplications() + + " #queue-pending-applications: " + getNumNonSchedulableApplications() + + " #queue-active-applications: " + getNumSchedulableApplications() ); } @@ -780,10 +806,10 @@ public synchronized void removeApplication(FiCaSchedulerApp application, User us " appId: " + application.getApplicationId() + " user: " + application.getUser() + " queue: " + getQueueName() + - " #user-pending-applications: " + user.getPendingApplications() + - " #user-active-applications: " + user.getActiveApplications() + - " #queue-pending-applications: " + getNumPendingApplications() + - " #queue-active-applications: " + getNumActiveApplications() + " #user-pending-applications: " + user.getNonSchedulableApplications() + + " #user-active-applications: " + user.getSchedulableApplications() + + " #queue-pending-applications: " + getNumNonSchedulableApplications() + + " #queue-active-applications: " + getNumSchedulableApplications() ); } @@ -964,7 +990,7 @@ private Resource computeUserLimitAndSetHeadroom( String user = application.getUser(); /** - * Headroom is min((userLimit, queue-max-cap) - consumed) + * Headroom is min(userLimit, queue-max-cap) - consumed */ Resource userLimit = // User limit @@ -1366,11 +1392,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod LOG.info("assignedContainer" + " application=" + application.getApplicationId() + " container=" + container + - " containerId=" + container.getId() + " queue=" + this + - " usedCapacity=" + getUsedCapacity() + - " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + - " used=" + usedResources + " cluster=" + clusterResource); return container.getResource(); @@ -1381,10 +1403,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod LOG.info("Reserved container " + " application=" + application.getApplicationId() + " resource=" + request.getCapability() + - " queue=" + this.toString() + - " usedCapacity=" + getUsedCapacity() + - " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + - " used=" + usedResources + + " queue=" + this + " cluster=" + clusterResource); return request.getCapability(); @@ -1450,11 +1469,7 @@ public void completedContainer(Resource clusterResource, application, container.getResource()); LOG.info("completedContainer" + " container=" + container + - " resource=" + container.getResource() + " queue=" + this + - " usedCapacity=" + getUsedCapacity() + - " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + - " used=" + usedResources + " cluster=" + clusterResource); // Inform the parent queue getParent().completedContainer(clusterResource, application, @@ -1553,40 +1568,40 @@ public QueueMetrics getMetrics() { static class User { Resource consumed = Resources.createResource(0, 0); - int pendingApplications = 0; - int activeApplications = 0; + int nonSchedulableApplications = 0; + int schedulableApplications = 0; public Resource getConsumedResources() { return consumed; } - public int getPendingApplications() { - return pendingApplications; + public int getNonSchedulableApplications() { + return nonSchedulableApplications; } - public int getActiveApplications() { - return activeApplications; + public int getSchedulableApplications() { + return schedulableApplications; } public int getTotalApplications() { - return getPendingApplications() + getActiveApplications(); + return getNonSchedulableApplications() + getSchedulableApplications(); } public synchronized void submitApplication() { - ++pendingApplications; + ++nonSchedulableApplications; } public synchronized void activateApplication() { - --pendingApplications; - ++activeApplications; + --nonSchedulableApplications; + ++schedulableApplications; } public synchronized void finishApplication(boolean wasActive) { if (wasActive) { - --activeApplications; + --schedulableApplications; } else { - --pendingApplications; + --nonSchedulableApplications; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java index 65c911b..7b99ca4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java @@ -30,16 +30,21 @@ public class UserInfo { protected String username; protected ResourceInfo resourcesUsed; - protected int numPendingApplications; - protected int numActiveApplications; + protected int numNonSchedulableApps; + protected int numSchedulableApps; + protected int numRunningApps; + protected int numPendingApps; UserInfo() {} - UserInfo(String username, Resource resUsed, int activeApps, int pendingApps) { + UserInfo(String username, Resource resUsed, int schedulableApps, + int nonSchedulableApps, int runningApps, int pendingApps) { this.username = username; this.resourcesUsed = new ResourceInfo(resUsed); - this.numActiveApplications = activeApps; - this.numPendingApplications = pendingApps; + this.numSchedulableApps = schedulableApps; + this.numNonSchedulableApps = nonSchedulableApps; + this.numRunningApps = runningApps; + this.numPendingApps = pendingApps; } public String getUsername() { @@ -50,11 +55,19 @@ public ResourceInfo getResourcesUsed() { return resourcesUsed; } - public int getNumPendingApplications() { - return numPendingApplications; + public int getNumNonSchedulableApplications() { + return numNonSchedulableApps; + } + + public int getNumSchedulableApplications() { + return numSchedulableApps; } - public int getNumActiveApplications() { - return numActiveApplications; + public int getNumRunningApplications() { + return numRunningApps; + } + + public int getNumPendingApplications() { + return numPendingApps; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 0bf8517..0a0eb42 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -98,30 +98,32 @@ protected void render(Block html) { for (UserInfo entry: users) { activeUserList.append(entry.getUsername()).append(" <") .append(getPercentage(entry.getResourcesUsed(), usedResources)) - .append(", Active Apps: " + entry.getNumActiveApplications()) + .append(", Non-schedulable Apps: " + entry.getNumNonSchedulableApplications()) .append(", Pending Apps: " + entry.getNumPendingApplications()) + .append(", Running Apps: " + entry.getNumRunningApplications()) .append(">
"); //Force line break } ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). _("Queue State:", lqinfo.getQueueState()). _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)). - _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)). - _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). + _("Used Capacity Across the Cluster:", percent(lqinfo.getAbsoluteUsedCapacity()/ 100)). _("Used Resources:", StringEscapeUtils.escapeHtml(lqinfo.getUsedResources().toString())). - _("Num Active Applications:", Integer.toString(lqinfo.getNumActiveApplications())). - _("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())). _("Num Containers:", Integer.toString(lqinfo.getNumContainers())). - _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())). - _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). - _("Max Active Applications:", Integer.toString(lqinfo.getMaxActiveApplications())). - _("Max Active Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())). + _("Num Non-schedulable Applications:", Integer.toString(lqinfo.getNumNonSchedulableApplications())). + _("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())). + _("Num Running Applications:", Integer.toString(lqinfo.getNumRunningApplications())). _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)). + _("Configured Capacity Across the Cluster:", percent(lqinfo.getAbsoluteCapacity() / 100)). _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). + _("Configured Max Capacity Across the Cluster:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). + _("Max Schedulable Applications:", Integer.toString(lqinfo.getMaxActiveApplications())). + _("Max Schedulable Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())). + _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())). + _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())). _r("Active users: ", activeUserList.toString()); - html._(InfoBlock.class); // clear the info contents so this queue's info doesn't accumulate into another queue's info diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index d90e963..47c6474 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -27,7 +27,9 @@ @XmlAccessorType(XmlAccessType.FIELD) public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { - protected int numActiveApplications; + protected int numSchedulableApplications; + protected int numNonSchedulableApplications; + protected int numRunningApplications; protected int numPendingApplications; protected int numContainers; protected int maxApplications; @@ -43,7 +45,9 @@ CapacitySchedulerLeafQueueInfo(LeafQueue q) { super(q); - numActiveApplications = q.getNumActiveApplications(); + numSchedulableApplications = q.getNumSchedulableApplications(); + numNonSchedulableApplications = q.getNumNonSchedulableApplications(); + numRunningApplications = q.getNumRunningApplications(); numPendingApplications = q.getNumPendingApplications(); numContainers = q.getNumContainers(); maxApplications = q.getMaxApplications(); @@ -55,8 +59,16 @@ userLimitFactor = q.getUserLimitFactor(); } - public int getNumActiveApplications() { - return numActiveApplications; + public int getSchedulableApplications() { + return numSchedulableApplications; + } + + public int getNumNonSchedulableApplications() { + return numNonSchedulableApplications; + } + + public int getNumRunningApplications() { + return numRunningApplications; } public int getNumPendingApplications() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f343bd5..a9682a2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -305,41 +305,41 @@ public void testActiveApplicationLimits() throws Exception { // Submit first application FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_0, user_0, A); - assertEquals(1, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(1, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(1, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); // Submit second application FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_1, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); // Submit third application, should remain pending FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_2, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); // Finish one application, app_2 should be activated queue.finishApplication(app_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); // Submit another one for user_0 FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_3, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); // Change queue limit to be smaller so 2 users can fill it up doReturn(3).when(queue).getMaximumActiveApplications(); @@ -347,31 +347,31 @@ public void testActiveApplicationLimits() throws Exception { // Submit first app for user_1 FiCaSchedulerApp app_4 = getMockApplication(APPLICATION_ID++, user_1); queue.submitApplication(app_4, user_1, A); - assertEquals(3, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); - assertEquals(1, queue.getNumActiveApplications(user_1)); - assertEquals(0, queue.getNumPendingApplications(user_1)); + assertEquals(3, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications(user_1)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_1)); // Submit second app for user_1, should block due to queue-limit FiCaSchedulerApp app_5 = getMockApplication(APPLICATION_ID++, user_1); queue.submitApplication(app_5, user_1, A); - assertEquals(3, queue.getNumActiveApplications()); - assertEquals(2, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); - assertEquals(1, queue.getNumActiveApplications(user_1)); - assertEquals(1, queue.getNumPendingApplications(user_1)); + assertEquals(3, queue.getNumSchedulableApplications()); + assertEquals(2, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications(user_1)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_1)); // Now finish one app of user_1 so app_5 should be activated queue.finishApplication(app_4, A); - assertEquals(3, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); - assertEquals(1, queue.getNumActiveApplications(user_1)); - assertEquals(0, queue.getNumPendingApplications(user_1)); + assertEquals(3, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications(user_1)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_1)); } @Test @@ -386,72 +386,72 @@ public void testActiveLimitsWithKilledApps() throws Exception { // Submit first application FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_0, user_0, A); - assertEquals(1, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(1, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(1, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); assertTrue(queue.activeApplications.contains(app_0)); // Submit second application FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_1, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); assertTrue(queue.activeApplications.contains(app_1)); // Submit third application, should remain pending FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_2, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); assertTrue(queue.pendingApplications.contains(app_2)); // Submit fourth application, should remain pending FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0); queue.submitApplication(app_3, user_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(2, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(2, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(2, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(2, queue.getNumNonSchedulableApplications(user_0)); assertTrue(queue.pendingApplications.contains(app_3)); // Kill 3rd pending application queue.finishApplication(app_2, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(1, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(1, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(1, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(1, queue.getNumNonSchedulableApplications(user_0)); assertFalse(queue.pendingApplications.contains(app_2)); assertFalse(queue.activeApplications.contains(app_2)); // Finish 1st application, app_3 should become active queue.finishApplication(app_0, A); - assertEquals(2, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(2, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(2, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(2, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); assertTrue(queue.activeApplications.contains(app_3)); assertFalse(queue.pendingApplications.contains(app_3)); assertFalse(queue.activeApplications.contains(app_0)); // Finish 2nd application queue.finishApplication(app_1, A); - assertEquals(1, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(1, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(1, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(1, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); assertFalse(queue.activeApplications.contains(app_1)); // Finish 4th application queue.finishApplication(app_3, A); - assertEquals(0, queue.getNumActiveApplications()); - assertEquals(0, queue.getNumPendingApplications()); - assertEquals(0, queue.getNumActiveApplications(user_0)); - assertEquals(0, queue.getNumPendingApplications(user_0)); + assertEquals(0, queue.getNumSchedulableApplications()); + assertEquals(0, queue.getNumNonSchedulableApplications()); + assertEquals(0, queue.getNumSchedulableApplications(user_0)); + assertEquals(0, queue.getNumNonSchedulableApplications(user_0)); assertFalse(queue.activeApplications.contains(app_3)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4f4bf2f..06d5da7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -353,6 +353,8 @@ public void testAppAttemptMetrics() throws Exception { cs.handle(event); assertEquals(0, a.getMetrics().getAppsPending()); + assertEquals(0, a.getNumPendingApplications()); + assertEquals(0, a.getNumPendingApplications(user_0)); assertEquals(1, a.getMetrics().getAppsFailed()); // Attempt the same application again @@ -364,7 +366,9 @@ public void testAppAttemptMetrics() throws Exception { assertEquals(1, a.getMetrics().getAppsSubmitted()); assertEquals(1, a.getMetrics().getAppsPending()); - + assertEquals(1, a.getNumPendingApplications()); + assertEquals(1, a.getNumPendingApplications(user_0)); + when(cs.getApplication(appAttemptId_1)).thenReturn(app_0); event = new AppRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED); @@ -372,6 +376,8 @@ public void testAppAttemptMetrics() throws Exception { assertEquals(1, a.getMetrics().getAppsSubmitted()); assertEquals(0, a.getMetrics().getAppsPending()); + assertEquals(0, a.getNumPendingApplications()); + assertEquals(0, a.getNumPendingApplications(user_0)); assertEquals(0, a.getMetrics().getAppsFailed()); assertEquals(1, a.getMetrics().getAppsCompleted());