diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8e65e6a42e3..e513cda6218 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -102,11 +102,13 @@ private final Map applicationSchedulingEnvs = new HashMap<>(); private final RMContext rmContext; private final int retryAttempts; + private boolean isUnmanagedAM; public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, AbstractUsersManager abstractUsersManager, long epoch, ResourceUsage appResourceUsage, - Map applicationSchedulingEnvs, RMContext rmContext) { + Map applicationSchedulingEnvs, RMContext rmContext, + boolean isUnmanagedAM) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; @@ -120,6 +122,7 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, this.retryAttempts = rmContext.getYarnConfiguration().getInt( YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS); + this.isUnmanagedAM = isUnmanagedAM; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); updateContext = new ContainerUpdateContext(this); @@ -156,6 +159,14 @@ public boolean isPending() { return pending; } + public void setUnmanagedAM(boolean isUAM) { + this.isUnmanagedAM = isUAM; + } + + public boolean isUnmanagedAM() { + return isUnmanagedAM; + } + public Set getRequestedPartitions() { return requestedPartitions; } @@ -617,8 +628,15 @@ public void move(Queue newQueue) { ap.getPrimaryRequestedNodePartition(), delta); } } - oldMetrics.moveAppFrom(this); - newMetrics.moveAppTo(this); + + if(isUnmanagedAM()) { + oldMetrics.moveUnmanagedAppFrom(this); + newMetrics.moveUnmanagedAppTo(this); + } else { + oldMetrics.moveAppFrom(this); + newMetrics.moveAppTo(this); + } + abstractUsersManager.deactivateApplication(user, applicationId); abstractUsersManager = newQueue.getAbstractUsersManager(); if (!schedulerKeys.isEmpty()) { @@ -649,7 +667,12 @@ public void stop() { ask.getCount())); } } - metrics.finishAppAttempt(applicationId, pending, user); + + if(isUnmanagedAM) { + metrics.finishUnmanagedAppAttempt(applicationId, pending, user); + } else { + metrics.finishAppAttempt(applicationId, pending, user); + } // Clear requests themselves clearRequests(); @@ -695,7 +718,11 @@ public void recoverContainer(RMContainer rmContainer, String partition) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + if(isUnmanagedAM()) { + metrics.runUnmanagedAppAttempt(applicationId, user); + } else { + metrics.runAppAttempt(applicationId, user); + } } // Container is completed. Skip recovering resources. @@ -736,7 +763,11 @@ private void updateMetricsForAllocatedContainer(NodeType type, // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + if(isUnmanagedAM()) { + metrics.runUnmanagedAppAttempt(applicationId, user); + } else { + metrics.runAppAttempt(applicationId, user); + } } updateMetrics(applicationId, type, node, containerAllocated, user, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index c3d466a8354..2e92af1c2f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -63,6 +63,20 @@ @Metric("# of apps completed") MutableCounterInt appsCompleted; @Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps failed") MutableCounterInt appsFailed; + + @Metric("# of Unmanaged apps submitted") + private MutableCounterInt unmanagedAppsSubmitted; + @Metric("# of Unmanaged running apps") + private MutableGaugeInt unmanagedAppsRunning; + @Metric("# of Unmanaged pending apps") + private MutableGaugeInt unmanagedAppsPending; + @Metric("# of Unmanaged apps completed") + private MutableCounterInt unmanagedAppsCompleted; + @Metric("# of Unmanaged apps killed") + private MutableCounterInt unmanagedAppsKilled; + @Metric("# of Unmanaged apps failed") + private MutableCounterInt unmanagedAppsFailed; + @Metric("Aggregate # of allocated node-local containers") MutableCounterLong aggregateNodeLocalContainersAllocated; @Metric("Aggregate # of allocated rack-local containers") @@ -413,6 +427,17 @@ public void submitApp(String user) { } } + public void submitUnmanagedApp(String user) { + unmanagedAppsSubmitted.incr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.submitUnmanagedApp(user); + } + if (parent != null) { + parent.submitUnmanagedApp(user); + } + } + public void submitAppAttempt(String user) { appsPending.incr(); QueueMetrics userMetrics = getUserMetrics(user); @@ -424,6 +449,17 @@ public void submitAppAttempt(String user) { } } + public void submitUnmanagedAppAttempt(String user) { + unmanagedAppsPending.incr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.submitUnmanagedAppAttempt(user); + } + if (parent != null) { + parent.submitUnmanagedAppAttempt(user); + } + } + public void runAppAttempt(ApplicationId appId, String user) { runBuckets.add(appId, System.currentTimeMillis()); appsRunning.incr(); @@ -437,6 +473,19 @@ public void runAppAttempt(ApplicationId appId, String user) { } } + public void runUnmanagedAppAttempt(ApplicationId appId, String user) { + runBuckets.add(appId, System.currentTimeMillis()); + unmanagedAppsRunning.incr(); + unmanagedAppsPending.decr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.runUnmanagedAppAttempt(appId, user); + } + if (parent != null) { + parent.runUnmanagedAppAttempt(appId, user); + } + } + public void finishAppAttempt( ApplicationId appId, boolean isPending, String user) { runBuckets.remove(appId); @@ -454,6 +503,23 @@ public void finishAppAttempt( } } + public void finishUnmanagedAppAttempt( + ApplicationId appId, boolean isPending, String user) { + runBuckets.remove(appId); + if (isPending) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.finishUnmanagedAppAttempt(appId, isPending, user); + } + if (parent != null) { + parent.finishUnmanagedAppAttempt(appId, isPending, user); + } + } + public void finishApp(String user, RMAppState rmAppFinalState) { switch (rmAppFinalState) { case KILLED: appsKilled.incr(); break; @@ -468,7 +534,22 @@ public void finishApp(String user, RMAppState rmAppFinalState) { parent.finishApp(user, rmAppFinalState); } } - + + public void finishUnmanagedApp(String user, RMAppState rmAppFinalState) { + switch (rmAppFinalState) { + case KILLED: unmanagedAppsKilled.incr(); break; + case FAILED: unmanagedAppsFailed.incr(); break; + default: unmanagedAppsCompleted.incr(); break; + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.finishUnmanagedApp(user, rmAppFinalState); + } + if (parent != null) { + parent.finishUnmanagedApp(user, rmAppFinalState); + } + } + public void moveAppFrom(AppSchedulingInfo app) { if (app.isPending()) { appsPending.decr(); @@ -483,7 +564,22 @@ public void moveAppFrom(AppSchedulingInfo app) { parent.moveAppFrom(app); } } - + + public void moveUnmanagedAppFrom(AppSchedulingInfo app) { + if (app.isPending()) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + QueueMetrics userMetrics = getUserMetrics(app.getUser()); + if (userMetrics != null) { + userMetrics.moveUnmanagedAppFrom(app); + } + if (parent != null) { + parent.moveUnmanagedAppFrom(app); + } + } + public void moveAppTo(AppSchedulingInfo app) { if (app.isPending()) { appsPending.incr(); @@ -499,6 +595,21 @@ public void moveAppTo(AppSchedulingInfo app) { } } + public void moveUnmanagedAppTo(AppSchedulingInfo app) { + if (app.isPending()) { + unmanagedAppsPending.incr(); + } else { + unmanagedAppsRunning.incr(); + } + QueueMetrics userMetrics = getUserMetrics(app.getUser()); + if (userMetrics != null) { + userMetrics.moveUnmanagedAppTo(app); + } + if (parent != null) { + parent.moveUnmanagedAppTo(app); + } + } + /** * Set available resources. To be called by scheduler periodically as * resources become available. @@ -1054,6 +1165,10 @@ public int getAppsSubmitted() { return appsSubmitted.value(); } + public int getUnmanagedAppsSubmitted() { + return unmanagedAppsSubmitted.value(); + } + public int getAppsRunning() { return appsRunning.value(); } @@ -1062,10 +1177,18 @@ public int getAppsPending() { return appsPending.value(); } + public int getUnmanagedAppsPending() { + return unmanagedAppsPending.value(); + } + public int getAppsCompleted() { return appsCompleted.value(); } + public int getUnmanagedAppsCompleted() { + return unmanagedAppsCompleted.value(); + } + public int getAppsKilled() { return appsKilled.value(); } @@ -1074,6 +1197,10 @@ public int getAppsFailed() { return appsFailed.value(); } + public int getUnmanagedAppsFailed() { + return unmanagedAppsFailed.value(); + } + public Resource getAllocatedResources() { if (queueMetricsForCustomResources != null) { return Resource.newInstance(allocatedMB.value(), allocatedVCores.value(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 96288f8b9f3..3e792918a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -30,16 +30,20 @@ private final String user; private volatile T currentAttempt; private volatile Priority priority; + private boolean isAppOnUAM; - public SchedulerApplication(Queue queue, String user) { + public SchedulerApplication(Queue queue, String user, boolean isAppOnUAM) { this.queue = queue; this.user = user; + this.isAppOnUAM = isAppOnUAM; this.priority = null; } - public SchedulerApplication(Queue queue, String user, Priority priority) { + public SchedulerApplication(Queue queue, String user, Priority priority, + boolean isAppOnUAM) { this.queue = queue; this.user = user; + this.isAppOnUAM = isAppOnUAM; this.priority = priority; } @@ -64,7 +68,11 @@ public void setCurrentAppAttempt(T currentAttempt) { } public void stop(RMAppState rmAppFinalState) { - queue.getMetrics().finishApp(user, rmAppFinalState); + if(isAppOnUAM()) { + queue.getMetrics().finishUnmanagedApp(user, rmAppFinalState); + } else { + queue.getMetrics().finishApp(user, rmAppFinalState); + } } public Priority getPriority() { @@ -80,4 +88,7 @@ public void setPriority(Priority priority) { } } + public boolean isAppOnUAM() { + return isAppOnUAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index e9575b9d5dd..b60aac5ffb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -241,7 +241,7 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage, - applicationSchedulingEnvs, rmContext); + applicationSchedulingEnvs, rmContext, unmanagedAM); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index a632bfa7651..3fc21d17301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -821,7 +821,8 @@ public boolean isAmbiguous(String queueName) { private void addApplicationOnRecovery(ApplicationId applicationId, String queueName, String user, - Priority priority, ApplicationPlacementContext placementContext) { + Priority priority, ApplicationPlacementContext placementContext, + boolean isAppOnUAM) { writeLock.lock(); try { //check if the queue needs to be auto-created during recovery @@ -883,9 +884,14 @@ private void addApplicationOnRecovery(ApplicationId applicationId, // Ignore the exception for recovered app as the app was previously // accepted. } - queue.getMetrics().submitApp(user); + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + isAppOnUAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -943,7 +949,7 @@ private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId private void addApplication(ApplicationId applicationId, String queueName, String user, Priority priority, - ApplicationPlacementContext placementContext) { + ApplicationPlacementContext placementContext, boolean isAppOnUAM) { writeLock.lock(); try { if (isSystemAppsLimitReached()) { @@ -1047,9 +1053,14 @@ private void addApplication(ApplicationId applicationId, String queueName, return; } // update the metrics - queue.getMetrics().submitApp(user); + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + isAppOnUAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -1851,11 +1862,11 @@ public void handle(SchedulerEvent event) { if (!appAddedEvent.getIsAppRecovering()) { addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), appAddedEvent.isAppOnUAM()); } else { addApplicationOnRecovery(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), appAddedEvent.isAppOnUAM()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 96d309c547e..c5b13ea5fa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -602,7 +602,12 @@ public void submitApplicationAttempt(FiCaSchedulerApp application, // We don't want to update metrics for move app if (!isMoveApp) { - metrics.submitAppAttempt(userName); + if(application.getAppSchedulingInfo() != null && + application.getAppSchedulingInfo().isUnmanagedAM()) { + metrics.submitUnmanagedAppAttempt(userName); + } else { + metrics.submitAppAttempt(userName); + } } getParent().submitApplicationAttempt(application, userName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 80b7f2fb821..783b8b02c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -34,6 +34,7 @@ private final boolean isAppRecovering; private final Priority appPriority; private final ApplicationPlacementContext placementContext; + private boolean isAppOnUAM = false; public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user) { @@ -58,6 +59,7 @@ public AppAddedSchedulerEvent(String user, this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, null); + this.isAppOnUAM = submissionContext.getUnmanagedAM(); } public AppAddedSchedulerEvent(String user, @@ -66,6 +68,7 @@ public AppAddedSchedulerEvent(String user, this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, placementContext); + this.isAppOnUAM = submissionContext.getUnmanagedAM(); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, @@ -108,4 +111,8 @@ public Priority getApplicatonPriority() { public ApplicationPlacementContext getPlacementContext() { return placementContext; } + + public boolean isAppOnUAM() { + return isAppOnUAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 813f87bd779..ba8b3f78fec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -552,11 +552,19 @@ protected void addApplication(ApplicationId applicationId, return; } } + boolean isAppOnUAM = rmApp != null && + rmApp.getApplicationSubmissionContext() != null + && rmApp.getApplicationSubmissionContext().getUnmanagedAM(); SchedulerApplication application = - new SchedulerApplication<>(queue, user); + new SchedulerApplication<>(queue, user, isAppOnUAM); applications.put(applicationId, application); - queue.getMetrics().submitApp(user); + + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName @@ -609,8 +617,11 @@ protected void addApplicationAttempt( } else{ maxRunningEnforcer.trackNonRunnableApp(attempt); } - - queue.getMetrics().submitAppAttempt(user); + if(application.isAppOnUAM()) { + queue.getMetrics().submitUnmanagedAppAttempt(user); + } else { + queue.getMetrics().submitAppAttempt(user); + } LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user: " + user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 9ec4822e513..994a721080d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -389,11 +389,15 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, @VisibleForTesting public synchronized void addApplication(ApplicationId applicationId, - String queue, String user, boolean isAppRecovering) { + String queue, String user, boolean isAppRecovering, boolean isAppOnUAM) { SchedulerApplication application = - new SchedulerApplication<>(DEFAULT_QUEUE, user); + new SchedulerApplication<>(DEFAULT_QUEUE, user, isAppOnUAM); applications.put(applicationId, application); - metrics.submitApp(user); + if(isAppOnUAM) { + metrics.submitUnmanagedApp(user); + } else { + metrics.submitApp(user); + } LOG.info("Accepted application " + applicationId + " from user: " + user + ", currently num of applications: " + applications.size()); if (isAppRecovering) { @@ -424,7 +428,11 @@ public synchronized void addApplication(ApplicationId applicationId, } application.setCurrentAppAttempt(schedulerApp); - metrics.submitAppAttempt(user); + if(application.isAppOnUAM()) { + metrics.submitUnmanagedAppAttempt(user); + } else { + metrics.submitAppAttempt(user); + } LOG.info("Added Application Attempt " + appAttemptId + " to scheduler from user " + application.getUser()); if (isAttemptRecovering) { @@ -768,8 +776,8 @@ public void handle(SchedulerEvent event) { { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; addApplication(appAddedEvent.getApplicationId(), - appAddedEvent.getQueue(), appAddedEvent.getUser(), - appAddedEvent.getIsAppRecovering()); + appAddedEvent.getQueue(), appAddedEvent.getUser(), + appAddedEvent.getIsAppRecovering(), appAddedEvent.isAppOnUAM()); } break; case APP_REMOVED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java index b9506be4b3e..19ee08f17cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java @@ -33,6 +33,12 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_PENDING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_RUNNING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_SUBMITTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_COMPLETED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_FAILED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_KILLED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_PENDING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_RUNNING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_SUBMITTED; final class AppMetricsChecker { private final static Logger LOG = @@ -45,7 +51,13 @@ .gaugeInt(APPS_RUNNING, 0) .counter(APPS_COMPLETED, 0) .counter(APPS_FAILED, 0) - .counter(APPS_KILLED, 0); + .counter(APPS_KILLED, 0) + .counter(UNMANAGED_APPS_SUBMITTED, 0) + .gaugeInt(UNMANAGED_APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .counter(UNMANAGED_APPS_COMPLETED, 0) + .counter(UNMANAGED_APPS_FAILED, 0) + .counter(UNMANAGED_APPS_KILLED, 0); enum AppMetricsKey { APPS_SUBMITTED("AppsSubmitted"), @@ -53,7 +65,13 @@ APPS_RUNNING("AppsRunning"), APPS_COMPLETED("AppsCompleted"), APPS_FAILED("AppsFailed"), - APPS_KILLED("AppsKilled"); + APPS_KILLED("AppsKilled"), + UNMANAGED_APPS_SUBMITTED("UnmanagedAppsSubmitted"), + UNMANAGED_APPS_PENDING("UnmanagedAppsPending"), + UNMANAGED_APPS_RUNNING("UnmanagedAppsRunning"), + UNMANAGED_APPS_COMPLETED("UnmanagedAppsCompleted"), + UNMANAGED_APPS_FAILED("UnmanagedAppsFailed"), + UNMANAGED_APPS_KILLED("UnmanagedAppsKilled"); private String value; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index b1a69f019bf..47174a47b05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -405,7 +405,8 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { RMApp mockAPp = new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW); SchedulerApplication application = - new SchedulerApplication(null, mockAPp.getUser()); + new SchedulerApplication(null, mockAPp.getUser(), + false); // Second app with one app attempt RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java index 62e5f3d72ab..391649d4092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java @@ -50,7 +50,7 @@ public void testBacklistChanged() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo(appAttemptId, "test", queue, null, 0, new ResourceUsage(), - new HashMap(), rmContext); + new HashMap(), rmContext, false); appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), new ArrayList()); @@ -124,7 +124,7 @@ public void testSchedulerKeyAccounting() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo info = new AppSchedulingInfo( appAttemptId, "test", queue, mock(ActiveUsersManager.class), 0, - new ResourceUsage(), new HashMap<>(), rmContext); + new ResourceUsage(), new HashMap<>(), rmContext, false); Assert.assertEquals(0, info.getSchedulerKeys().size()); Priority pri1 = Priority.newInstance(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 33c39290de5..cf46eefa8fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -44,6 +44,10 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_PENDING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_RUNNING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_SUBMITTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_FAILED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_PENDING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_RUNNING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_SUBMITTED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_ALLOCATED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_RELEASED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.ALLOCATED_CONTAINERS; @@ -243,6 +247,87 @@ public void testQueueAppMetricsForMultipleFailures() { assertNull(userSource); } + @Test + public void testQueueUnmanagedAppMetricsForMultipleFailures() { + String queueName = "single"; + + QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, + new Configuration()); + MetricsSource queueSource = queueSource(ms, queueName); + AppSchedulingInfo app = mockApp(USER); + + // Submit an unmanaged Application. + metrics.submitUnmanagedApp(USER); + MetricsSource userSource = userSource(ms, queueName, USER); + AppMetricsChecker appMetricsChecker = AppMetricsChecker.create() + .counter(UNMANAGED_APPS_SUBMITTED, 1) + .checkAgainst(queueSource, true); + metrics.submitUnmanagedAppAttempt(USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runUnmanagedAppAttempt(app.getApplicationId(), USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + metrics.finishUnmanagedAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser()); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + // As the application has failed, framework retries the same application + // based on configuration + metrics.submitUnmanagedAppAttempt(USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runUnmanagedAppAttempt(app.getApplicationId(), USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + // Suppose say application has failed this time as well. + metrics.finishUnmanagedAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser()); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + // As the application has failed, framework retries the same application + // based on configuration + metrics.submitUnmanagedAppAttempt(USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runUnmanagedAppAttempt(app.getApplicationId(), USER); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + // Suppose say application has failed, and there's no more retries. + metrics.finishUnmanagedAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser()); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + metrics.finishUnmanagedApp(USER, RMAppState.FAILED); + AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .counter(UNMANAGED_APPS_FAILED, 1) + .checkAgainst(queueSource, true); + + assertNull(userSource); + } + @Test public void testSingleQueueWithUserMetrics() { String queueName = "single2"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index b1080f7d5fe..94b6edc99ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -126,6 +126,7 @@ public void testMove() { when(rmContext.getYarnConfiguration()).thenReturn(conf); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.getAbstractUsersManager(), rmContext); + app.appSchedulingInfo.setUnmanagedAM(false); oldMetrics.submitApp(user); // confirm that containerId is calculated based on epoch. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index e7abf7d53df..d2ee5b2837e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -53,6 +53,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -131,6 +135,7 @@ CapacityScheduler cs; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + RMApp rmApp; CSQueue root; private CSQueueStore queues; @@ -170,7 +175,7 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) ConcurrentMap spyApps = spy(new ConcurrentHashMap()); - RMApp rmApp = mock(RMApp.class); + rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt(any())).thenReturn(null); amResourceRequest = mock(ResourceRequest.class); when(amResourceRequest.getCapability()).thenReturn( @@ -463,7 +468,10 @@ public void testPolicyConfiguration() throws Exception { public void testAppAttemptMetrics() throws Exception { CSMaxRunningAppsEnforcer enforcer = mock(CSMaxRunningAppsEnforcer.class); cs.setMaxRunningAppsEnforcer(enforcer); - + ApplicationSubmissionContext applicationSubmissionContext = mock(ApplicationSubmissionContext.class); + when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(false); + when(rmApp.getApplicationSubmissionContext()).thenReturn(applicationSubmissionContext); + when(rmApp.getCurrentAppAttempt()).thenReturn(mock(RMAppAttempt.class)); // Manipulate queue 'a' LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); @@ -494,6 +502,7 @@ public void testAppAttemptMetrics() throws Exception { .getMockApplicationAttemptId(0, 2); FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null, spyRMContext); + app_1.getAppSchedulingInfo().setUnmanagedAM(false); app_1.setAMResource(Resource.newInstance(100, 1)); a.submitApplicationAttempt(app_1, user_0); // same user @@ -521,6 +530,74 @@ public void testAppAttemptMetrics() throws Exception { assertEquals(1, userMetrics.getAppsSubmitted()); } + @Test + public void testUnmanagedAppAttemptMetrics() throws Exception { + CSMaxRunningAppsEnforcer enforcer = mock(CSMaxRunningAppsEnforcer.class); + cs.setMaxRunningAppsEnforcer(enforcer); + // Manipulate queue 'a' + LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); + + // Users + final String user_0 = "user_0"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = TestUtils + .getMockApplicationAttemptId(0, 1); + + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + appAttemptId_0.getApplicationId(), "test", a.getQueuePath(), + Priority.newInstance(0), null, true, true, + 2, null, "test"); + + AppAddedSchedulerEvent addAppEvent = + new AppAddedSchedulerEvent(user_0, applicationSubmissionContext, false, + null); + cs.handle(addAppEvent); + AppAttemptAddedSchedulerEvent addAttemptEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId_0, false); + cs.handle(addAttemptEvent); + + AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent( + appAttemptId_0, RMAppAttemptState.FAILED, false); + cs.handle(event); + + assertEquals(0, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(0, a.getMetrics().getUnmanagedAppsFailed()); + + // Attempt the same application again + final ApplicationAttemptId appAttemptId_1 = TestUtils + .getMockApplicationAttemptId(0, 2); + FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, + null, spyRMContext); + + app_1.setAMResource(Resource.newInstance(100, 1)); + a.submitApplicationAttempt(app_1, user_0); // same user + + assertEquals(1, a.getMetrics().getUnmanagedAppsSubmitted()); + assertEquals(1, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(1, a.getUser(user_0).getActiveApplications()); + assertEquals(app_1.getAMResource().getMemorySize(), a.getMetrics() + .getUsedAMResourceMB()); + assertEquals(app_1.getAMResource().getVirtualCores(), a.getMetrics() + .getUsedAMResourceVCores()); + + event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, + RMAppAttemptState.FINISHED, false); + cs.handle(event); + AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent( + appAttemptId_0.getApplicationId(), RMAppState.FINISHED); + cs.handle(rEvent); + + assertEquals(1, a.getMetrics().getUnmanagedAppsSubmitted()); + assertEquals(0, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(0, a.getMetrics().getUnmanagedAppsFailed()); + assertEquals(1, a.getMetrics().getUnmanagedAppsCompleted()); + + QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0); + assertEquals(1, userMetrics.getUnmanagedAppsSubmitted()); + } + @Test public void testFairConfiguration() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 40650532ed2..7183f7f782f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -796,7 +796,7 @@ public void testNodeUpdateBeforeAppAttemptInit() throws Exception { scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); - scheduler.addApplication(appId, "queue1", "user1", false); + scheduler.addApplication(appId, "queue1", "user1", false, false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try {