diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index b5b22b6..188c6b5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -260,7 +260,7 @@ synchronized public void allocate(NodeType type, SchedulerNode node, // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; - metrics.incrAppsRunning(this, user); + metrics.runAppAttempt(this, user); } LOG.debug("allocate: user: " + user + ", memory: " + request.getCapability()); @@ -390,7 +390,7 @@ synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) { .getNumContainers())); } } - metrics.finishApp(this, rmAppAttemptFinalState); + metrics.finishAppAttempt(this); // Clear requests themselves clearRequests(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 8a03095..a2283b7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.slf4j.Logger; @@ -57,7 +57,7 @@ @Metric("# of pending apps") MutableGaugeInt appsPending; @Metric("# of apps completed") MutableCounterInt appsCompleted; @Metric("# of apps killed") MutableCounterInt appsKilled; - @Metric("# of apps failed") MutableGaugeInt appsFailed; + @Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB; @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; @@ -214,43 +214,43 @@ public void getMetrics(MetricsCollector collector, boolean all) { registry.snapshot(collector.addRecord(registry.info()), all); } - public void submitApp(String user, int attemptId) { - if (attemptId == 1) { - appsSubmitted.incr(); - } else { - appsFailed.decr(); + public void submitApp(String user) { + appsSubmitted.incr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.submitApp(user); } + if (parent != null) { + parent.submitApp(user); + } + } + + public void submitAppAttempt(String user) { appsPending.incr(); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.submitApp(user, attemptId); + userMetrics.submitAppAttempt(user); } if (parent != null) { - parent.submitApp(user, attemptId); + parent.submitAppAttempt(user); } } - public void incrAppsRunning(AppSchedulingInfo app, String user) { + public void runAppAttempt(AppSchedulingInfo app, String user) { runBuckets.add(app.getApplicationId(), System.currentTimeMillis()); appsRunning.incr(); appsPending.decr(); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.incrAppsRunning(app, user); + userMetrics.runAppAttempt(app, user); } if (parent != null) { - parent.incrAppsRunning(app, user); + parent.runAppAttempt(app, user); } } - public void finishApp(AppSchedulingInfo app, - RMAppAttemptState rmAppAttemptFinalState) { + public void finishAppAttempt(AppSchedulingInfo app) { runBuckets.remove(app.getApplicationId()); - switch (rmAppAttemptFinalState) { - case KILLED: appsKilled.incr(); break; - case FAILED: appsFailed.incr(); break; - default: appsCompleted.incr(); break; - } if (app.isPending()) { appsPending.decr(); } else { @@ -258,10 +258,25 @@ public void finishApp(AppSchedulingInfo app, } QueueMetrics userMetrics = getUserMetrics(app.getUser()); if (userMetrics != null) { - userMetrics.finishApp(app, rmAppAttemptFinalState); + userMetrics.finishAppAttempt(app); + } + if (parent != null) { + parent.finishAppAttempt(app); + } + } + + public void finishApp(String user, RMAppState rmAppFinalState) { + switch (rmAppFinalState) { + case KILLED: appsKilled.incr(); break; + case FAILED: appsFailed.incr(); break; + default: appsCompleted.incr(); break; + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.finishApp(user, rmAppFinalState); } if (parent != null) { - parent.finishApp(app, rmAppAttemptFinalState); + parent.finishApp(user, rmAppFinalState); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 48e3ee8..5331dfc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -19,6 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @Private @Unstable @@ -39,4 +40,9 @@ public Queue getQueue() { public String getUser() { return user; } + + public void stop(RMAppState rmAppFinalState) { + queue.getMetrics().finishApp(user, rmAppFinalState); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 5f34108..ab229ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -496,6 +496,7 @@ private synchronized void doneApplication(ApplicationId applicationId, } else { queue.finishApplication(applicationId, application.getUser()); } + application.stop(finalState); applications.remove(applicationId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a8581a0..9bc80bc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -644,8 +644,7 @@ public void submitApplicationAttempt(FiCaSchedulerApp application, addApplicationAttempt(application, user); } - int attemptId = application.getApplicationAttemptId().getAttemptId(); - metrics.submitApp(userName, attemptId); + metrics.submitAppAttempt(userName); getParent().submitApplicationAttempt(application, userName); } @@ -702,6 +701,8 @@ public void submitApplication(ApplicationId applicationId, String userName, getParent().getQueuePath(), ace); throw ace; } + + metrics.submitApp(userName); } private synchronized void activateApplications() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index bdfbcab..a7106bc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -638,6 +638,7 @@ protected synchronized void addApplication(ApplicationId applicationId, SchedulerApplication application = new SchedulerApplication(queue, user); applications.put(applicationId, application); + queue.getMetrics().submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -668,7 +669,7 @@ protected synchronized void addApplicationAttempt( maxRunningEnforcer.trackNonRunnableApp(schedulerApp); } - queue.getMetrics().submitApp(user, applicationAttemptId.getAttemptId()); + queue.getMetrics().submitAppAttempt(user); appAttempts.put(applicationAttemptId, schedulerApp); LOG.info("Added Application Attempt " + applicationAttemptId @@ -704,6 +705,11 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { private synchronized void removeApplication(ApplicationId applicationId, RMAppState finalState) { + SchedulerApplication application = applications.get(applicationId); + if (application == null){ + return; + } + application.stop(finalState); applications.remove(applicationId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 9d42913..b3943c6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -362,8 +362,9 @@ private FiCaSchedulerNode getNode(NodeId nodeId) { private synchronized void addApplication(ApplicationId applicationId, String queue, String user) { SchedulerApplication application = - new SchedulerApplication(null, user); + new SchedulerApplication(DEFAULT_QUEUE, user); applications.put(applicationId, application); + metrics.submitApp(user); LOG.info("Accepted application " + applicationId + " from user: " + user); rmContext.getDispatcher().getEventHandler() .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); @@ -379,7 +380,7 @@ private synchronized void addApplicationAttempt( new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE, activeUsersManager, this.rmContext); appAttempts.put(appAttemptId, schedulerApp); - metrics.submitApp(user, appAttemptId.getAttemptId()); + metrics.submitAppAttempt(user); LOG.info("Added Application Attempt " + appAttemptId + " to scheduler from user " + application.getUser() + ", currently active: " + appAttempts.size()); @@ -395,6 +396,7 @@ private synchronized void doneApplication(ApplicationId applicationId, // Inform the activeUsersManager activeUsersManager.deactivateApplication(application.getUser(), applicationId); + application.stop(finalState); applications.remove(applicationId); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index b3702ad..a92e209 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -66,8 +66,10 @@ public void setUp() { MetricsSource queueSource= queueSource(ms, queueName); AppSchedulingInfo app = mockApp(user); - metrics.submitApp(user, 1); + metrics.submitApp(user); MetricsSource userSource = userSource(ms, queueName, user); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); @@ -76,7 +78,7 @@ public void setUp() { // configurable cluster/queue resources checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2)); @@ -85,7 +87,9 @@ public void setUp() { metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); - metrics.finishApp(app, RMAppAttemptState.FINISHED); + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + metrics.finishApp(user, RMAppState.FINISHED); checkApps(queueSource, 1, 0, 0, 1, 0, 0, true); assertNull(userSource); } @@ -100,39 +104,44 @@ public void testQueueAppMetricsForMultipleFailures() { MetricsSource queueSource = queueSource(ms, queueName); AppSchedulingInfo app = mockApp(user); - metrics.submitApp(user, 1); + metrics.submitApp(user); MetricsSource userSource = userSource(ms, queueName, user); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); - metrics.finishApp(app, RMAppAttemptState.FAILED); - checkApps(queueSource, 1, 0, 0, 0, 1, 0, true); + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); // As the application has failed, framework retries the same application // based on configuration - metrics.submitApp(user, 2); + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); // Suppose say application has failed this time as well. - metrics.finishApp(app, RMAppAttemptState.FAILED); - checkApps(queueSource, 1, 0, 0, 0, 1, 0, true); + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); // As the application has failed, framework retries the same application // based on configuration - metrics.submitApp(user, 3); + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); - // Suppose say application has finished. - metrics.finishApp(app, RMAppAttemptState.FINISHED); - checkApps(queueSource, 1, 0, 0, 1, 0, 0, true); + // Suppose say application has failed, and there's no more retries. + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + + metrics.finishApp(user, RMAppState.FAILED); + checkApps(queueSource, 1, 0, 0, 0, 1, 0, true); assertNull(userSource); } @@ -146,9 +155,13 @@ public void testQueueAppMetricsForMultipleFailures() { MetricsSource queueSource = queueSource(ms, queueName); AppSchedulingInfo app = mockApp(user); - metrics.submitApp(user, 1); + metrics.submitApp(user); MetricsSource userSource = userSource(ms, queueName, user); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(userSource, 1, 0, 0, 0, 0, 0, true); + + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); checkApps(userSource, 1, 1, 0, 0, 0, 0, true); @@ -160,7 +173,7 @@ public void testQueueAppMetricsForMultipleFailures() { checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); @@ -172,7 +185,10 @@ public void testQueueAppMetricsForMultipleFailures() { checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); - metrics.finishApp(app, RMAppAttemptState.FINISHED); + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(userSource, 1, 0, 0, 0, 0, 0, true); + metrics.finishApp(user, RMAppState.FINISHED); checkApps(queueSource, 1, 0, 0, 1, 0, 0, true); checkApps(userSource, 1, 0, 0, 1, 0, 0, true); } @@ -192,10 +208,16 @@ public void testQueueAppMetricsForMultipleFailures() { MetricsSource queueSource = queueSource(ms, leafQueueName); AppSchedulingInfo app = mockApp(user); - metrics.submitApp(user, 1); + metrics.submitApp(user); MetricsSource userSource = userSource(ms, leafQueueName, user); MetricsSource parentUserSource = userSource(ms, parentQueueName, user); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(userSource, 1, 0, 0, 0, 0, 0, true); + checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); + + metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true); checkApps(userSource, 1, 1, 0, 0, 0, 0, true); @@ -211,7 +233,7 @@ public void testQueueAppMetricsForMultipleFailures() { checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); checkResources(parentUserSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); - metrics.incrAppsRunning(app, user); + metrics.runAppAttempt(app, user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); @@ -231,7 +253,13 @@ public void testQueueAppMetricsForMultipleFailures() { checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); checkResources(parentUserSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); - metrics.finishApp(app, RMAppAttemptState.FINISHED); + metrics.finishAppAttempt(app); + checkApps(queueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); + checkApps(userSource, 1, 0, 0, 0, 0, 0, true); + checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); + + metrics.finishApp(user, RMAppState.FINISHED); checkApps(queueSource, 1, 0, 0, 1, 0, 0, true); checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true); checkApps(userSource, 1, 0, 0, 1, 0, 0, true); @@ -308,7 +336,7 @@ public static void checkApps(MetricsSource source, int submitted, int pending, assertGauge("AppsPending", pending, rb); assertGauge("AppsRunning", running, rb); assertCounter("AppsCompleted", completed, rb); - assertGauge("AppsFailed", failed, rb); + assertCounter("AppsFailed", failed, rb); assertCounter("AppsKilled", killed, rb); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 5e272de..190c05c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -63,7 +64,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -345,6 +348,9 @@ public void testAppAttemptMetrics() throws Exception { .getMockApplicationAttemptId(0, 1); FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null, rmContext); + AppAddedSchedulerEvent aEvent = new AppAddedSchedulerEvent( + appAttemptId_0.getApplicationId(), a.getQueueName(), user_0); + cs.handle(aEvent); a.submitApplicationAttempt(app_0, user_0); when(cs.getApplication(appAttemptId_0)).thenReturn(app_0); @@ -353,7 +359,7 @@ public void testAppAttemptMetrics() throws Exception { cs.handle(event); assertEquals(0, a.getMetrics().getAppsPending()); - assertEquals(1, a.getMetrics().getAppsFailed()); + assertEquals(0, a.getMetrics().getAppsFailed()); // Attempt the same application again final ApplicationAttemptId appAttemptId_1 = TestUtils @@ -369,6 +375,9 @@ public void testAppAttemptMetrics() throws Exception { event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED); cs.handle(event); + AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent( + appAttemptId_0.getApplicationId(), RMAppState.FINISHED); + cs.handle(rEvent); assertEquals(1, a.getMetrics().getAppsSubmitted()); assertEquals(0, a.getMetrics().getAppsPending()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 3057826..d01687f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -160,11 +159,6 @@ public void testAppAttemptMetrics() throws Exception { schedular.handle(attemptEvent); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); - - SchedulerEvent appEvent2 = - new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue", - "user"); - schedular.handle(appEvent2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId); schedular.handle(attemptEvent2);