diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8e65e6a42e3..b1876432090 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -102,11 +102,13 @@ private final Map applicationSchedulingEnvs = new HashMap<>(); private final RMContext rmContext; private final int retryAttempts; + private boolean isUnmanagedAM; public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, AbstractUsersManager abstractUsersManager, long epoch, ResourceUsage appResourceUsage, - Map applicationSchedulingEnvs, RMContext rmContext) { + Map applicationSchedulingEnvs, RMContext rmContext, + boolean isUnmanagedAM) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; @@ -120,6 +122,7 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, this.retryAttempts = rmContext.getYarnConfiguration().getInt( YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS); + this.isUnmanagedAM = isUnmanagedAM; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); updateContext = new ContainerUpdateContext(this); @@ -156,6 +159,10 @@ public boolean isPending() { return pending; } + public boolean isUnmanagedAM() { + return isUnmanagedAM; + } + public Set getRequestedPartitions() { return requestedPartitions; } @@ -617,8 +624,15 @@ public void move(Queue newQueue) { ap.getPrimaryRequestedNodePartition(), delta); } } - oldMetrics.moveAppFrom(this); - newMetrics.moveAppTo(this); + + if(isUnmanagedAM()) { + oldMetrics.moveUnmanagedAppFrom(this); + newMetrics.moveUnmanagedAppTo(this); + } else { + oldMetrics.moveAppFrom(this); + newMetrics.moveAppTo(this); + } + abstractUsersManager.deactivateApplication(user, applicationId); abstractUsersManager = newQueue.getAbstractUsersManager(); if (!schedulerKeys.isEmpty()) { @@ -649,7 +663,12 @@ public void stop() { ask.getCount())); } } - metrics.finishAppAttempt(applicationId, pending, user); + + if(isUnmanagedAM) { + metrics.finishUnmanagedAppAttempt(applicationId, pending, user); + } else { + metrics.finishAppAttempt(applicationId, pending, user); + } // Clear requests themselves clearRequests(); @@ -695,7 +714,11 @@ public void recoverContainer(RMContainer rmContainer, String partition) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + if(isUnmanagedAM()) { + metrics.runUnmanagedAppAttempt(applicationId, user); + } else { + metrics.runAppAttempt(applicationId, user); + } } // Container is completed. Skip recovering resources. @@ -736,7 +759,11 @@ private void updateMetricsForAllocatedContainer(NodeType type, // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + if(isUnmanagedAM()) { + metrics.runUnmanagedAppAttempt(applicationId, user); + } else { + metrics.runAppAttempt(applicationId, user); + } } updateMetrics(applicationId, type, node, containerAllocated, user, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index c3d466a8354..e92a1023203 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -63,6 +63,20 @@ @Metric("# of apps completed") MutableCounterInt appsCompleted; @Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps failed") MutableCounterInt appsFailed; + + @Metric("# of Unmanaged apps submitted") + MutableCounterInt unmanagedAppsSubmitted; + @Metric("# of Unmanaged running apps") + MutableGaugeInt unmanagedAppsRunning; + @Metric("# of Unmanaged pending apps") + MutableGaugeInt unmanagedAppsPending; + @Metric("# of Unmanaged apps completed") + MutableCounterInt unmanagedAppsCompleted; + @Metric("# of Unmanaged apps killed") + MutableCounterInt unmanagedAppsKilled; + @Metric("# of Unmanaged apps failed") + MutableCounterInt unmanagedAppsFailed; + @Metric("Aggregate # of allocated node-local containers") MutableCounterLong aggregateNodeLocalContainersAllocated; @Metric("Aggregate # of allocated rack-local containers") @@ -413,6 +427,17 @@ public void submitApp(String user) { } } + public void submitUnmanagedApp(String user) { + unmanagedAppsSubmitted.incr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.submitUnmanagedApp(user); + } + if (parent != null) { + parent.submitUnmanagedApp(user); + } + } + public void submitAppAttempt(String user) { appsPending.incr(); QueueMetrics userMetrics = getUserMetrics(user); @@ -424,6 +449,17 @@ public void submitAppAttempt(String user) { } } + public void submitUnmanagedAppAttempt(String user) { + unmanagedAppsPending.incr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.submitUnmanagedAppAttempt(user); + } + if (parent != null) { + parent.submitUnmanagedAppAttempt(user); + } + } + public void runAppAttempt(ApplicationId appId, String user) { runBuckets.add(appId, System.currentTimeMillis()); appsRunning.incr(); @@ -437,6 +473,19 @@ public void runAppAttempt(ApplicationId appId, String user) { } } + public void runUnmanagedAppAttempt(ApplicationId appId, String user) { + runBuckets.add(appId, System.currentTimeMillis()); + unmanagedAppsRunning.incr(); + unmanagedAppsPending.decr(); + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.runUnmanagedAppAttempt(appId, user); + } + if (parent != null) { + parent.runUnmanagedAppAttempt(appId, user); + } + } + public void finishAppAttempt( ApplicationId appId, boolean isPending, String user) { runBuckets.remove(appId); @@ -454,6 +503,23 @@ public void finishAppAttempt( } } + public void finishUnmanagedAppAttempt( + ApplicationId appId, boolean isPending, String user) { + runBuckets.remove(appId); + if (isPending) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.finishUnmanagedAppAttempt(appId, isPending, user); + } + if (parent != null) { + parent.finishUnmanagedAppAttempt(appId, isPending, user); + } + } + public void finishApp(String user, RMAppState rmAppFinalState) { switch (rmAppFinalState) { case KILLED: appsKilled.incr(); break; @@ -468,7 +534,22 @@ public void finishApp(String user, RMAppState rmAppFinalState) { parent.finishApp(user, rmAppFinalState); } } - + + public void finishUnmanagedApp(String user, RMAppState rmAppFinalState) { + switch (rmAppFinalState) { + case KILLED: unmanagedAppsKilled.incr(); break; + case FAILED: unmanagedAppsFailed.incr(); break; + default: unmanagedAppsCompleted.incr(); break; + } + QueueMetrics userMetrics = getUserMetrics(user); + if (userMetrics != null) { + userMetrics.finishUnmanagedApp(user, rmAppFinalState); + } + if (parent != null) { + parent.finishUnmanagedApp(user, rmAppFinalState); + } + } + public void moveAppFrom(AppSchedulingInfo app) { if (app.isPending()) { appsPending.decr(); @@ -483,7 +564,22 @@ public void moveAppFrom(AppSchedulingInfo app) { parent.moveAppFrom(app); } } - + + public void moveUnmanagedAppFrom(AppSchedulingInfo app) { + if (app.isPending()) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + QueueMetrics userMetrics = getUserMetrics(app.getUser()); + if (userMetrics != null) { + userMetrics.moveUnmanagedAppFrom(app); + } + if (parent != null) { + parent.moveUnmanagedAppFrom(app); + } + } + public void moveAppTo(AppSchedulingInfo app) { if (app.isPending()) { appsPending.incr(); @@ -499,6 +595,21 @@ public void moveAppTo(AppSchedulingInfo app) { } } + public void moveUnmanagedAppTo(AppSchedulingInfo app) { + if (app.isPending()) { + unmanagedAppsPending.incr(); + } else { + unmanagedAppsRunning.incr(); + } + QueueMetrics userMetrics = getUserMetrics(app.getUser()); + if (userMetrics != null) { + userMetrics.moveUnmanagedAppTo(app); + } + if (parent != null) { + parent.moveUnmanagedAppTo(app); + } + } + /** * Set available resources. To be called by scheduler periodically as * resources become available. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 96288f8b9f3..3e792918a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -30,16 +30,20 @@ private final String user; private volatile T currentAttempt; private volatile Priority priority; + private boolean isAppOnUAM; - public SchedulerApplication(Queue queue, String user) { + public SchedulerApplication(Queue queue, String user, boolean isAppOnUAM) { this.queue = queue; this.user = user; + this.isAppOnUAM = isAppOnUAM; this.priority = null; } - public SchedulerApplication(Queue queue, String user, Priority priority) { + public SchedulerApplication(Queue queue, String user, Priority priority, + boolean isAppOnUAM) { this.queue = queue; this.user = user; + this.isAppOnUAM = isAppOnUAM; this.priority = priority; } @@ -64,7 +68,11 @@ public void setCurrentAppAttempt(T currentAttempt) { } public void stop(RMAppState rmAppFinalState) { - queue.getMetrics().finishApp(user, rmAppFinalState); + if(isAppOnUAM()) { + queue.getMetrics().finishUnmanagedApp(user, rmAppFinalState); + } else { + queue.getMetrics().finishApp(user, rmAppFinalState); + } } public Priority getPriority() { @@ -80,4 +88,7 @@ public void setPriority(Priority priority) { } } + public boolean isAppOnUAM() { + return isAppOnUAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index e9575b9d5dd..b60aac5ffb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -241,7 +241,7 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage, - applicationSchedulingEnvs, rmContext); + applicationSchedulingEnvs, rmContext, unmanagedAM); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index a632bfa7651..3fc21d17301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -821,7 +821,8 @@ public boolean isAmbiguous(String queueName) { private void addApplicationOnRecovery(ApplicationId applicationId, String queueName, String user, - Priority priority, ApplicationPlacementContext placementContext) { + Priority priority, ApplicationPlacementContext placementContext, + boolean isAppOnUAM) { writeLock.lock(); try { //check if the queue needs to be auto-created during recovery @@ -883,9 +884,14 @@ private void addApplicationOnRecovery(ApplicationId applicationId, // Ignore the exception for recovered app as the app was previously // accepted. } - queue.getMetrics().submitApp(user); + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + isAppOnUAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -943,7 +949,7 @@ private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId private void addApplication(ApplicationId applicationId, String queueName, String user, Priority priority, - ApplicationPlacementContext placementContext) { + ApplicationPlacementContext placementContext, boolean isAppOnUAM) { writeLock.lock(); try { if (isSystemAppsLimitReached()) { @@ -1047,9 +1053,14 @@ private void addApplication(ApplicationId applicationId, String queueName, return; } // update the metrics - queue.getMetrics().submitApp(user); + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + isAppOnUAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -1851,11 +1862,11 @@ public void handle(SchedulerEvent event) { if (!appAddedEvent.getIsAppRecovering()) { addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), appAddedEvent.isAppOnUAM()); } else { addApplicationOnRecovery(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), appAddedEvent.isAppOnUAM()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 96d309c547e..1937a4e44c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -602,7 +602,11 @@ public void submitApplicationAttempt(FiCaSchedulerApp application, // We don't want to update metrics for move app if (!isMoveApp) { - metrics.submitAppAttempt(userName); + if(application.getAppSchedulingInfo().isUnmanagedAM()) { + metrics.submitUnmanagedAppAttempt(userName); + } else { + metrics.submitAppAttempt(userName); + } } getParent().submitApplicationAttempt(application, userName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 80b7f2fb821..793b9e14fcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -34,22 +34,24 @@ private final boolean isAppRecovering; private final Priority appPriority; private final ApplicationPlacementContext placementContext; + private final boolean isAppOnUAM; public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user) { this(applicationId, queue, user, false, null, Priority.newInstance(0), - null); + null, false); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, ApplicationPlacementContext placementContext) { this(applicationId, queue, user, false, null, Priority.newInstance(0), - placementContext); + placementContext, false); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, ReservationId reservationID, Priority appPriority) { - this(applicationId, queue, user, false, reservationID, appPriority, null); + this(applicationId, queue, user, false, reservationID, appPriority, null, + false); } public AppAddedSchedulerEvent(String user, @@ -57,7 +59,7 @@ public AppAddedSchedulerEvent(String user, Priority appPriority) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), - appPriority, null); + appPriority, null, submissionContext.getUnmanagedAM()); } public AppAddedSchedulerEvent(String user, @@ -65,12 +67,13 @@ public AppAddedSchedulerEvent(String user, Priority appPriority, ApplicationPlacementContext placementContext) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), - appPriority, placementContext); + appPriority, placementContext, submissionContext.getUnmanagedAM()); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, boolean isAppRecovering, ReservationId reservationID, - Priority appPriority, ApplicationPlacementContext placementContext) { + Priority appPriority, ApplicationPlacementContext placementContext, + boolean isAppOnUAM) { super(SchedulerEventType.APP_ADDED); this.applicationId = applicationId; this.queue = queue; @@ -79,6 +82,7 @@ public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, this.isAppRecovering = isAppRecovering; this.appPriority = appPriority; this.placementContext = placementContext; + this.isAppOnUAM = isAppOnUAM; } public ApplicationId getApplicationId() { @@ -108,4 +112,8 @@ public Priority getApplicatonPriority() { public ApplicationPlacementContext getPlacementContext() { return placementContext; } + + public boolean isAppOnUAM() { + return isAppOnUAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 813f87bd779..ba8b3f78fec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -552,11 +552,19 @@ protected void addApplication(ApplicationId applicationId, return; } } + boolean isAppOnUAM = rmApp != null && + rmApp.getApplicationSubmissionContext() != null + && rmApp.getApplicationSubmissionContext().getUnmanagedAM(); SchedulerApplication application = - new SchedulerApplication<>(queue, user); + new SchedulerApplication<>(queue, user, isAppOnUAM); applications.put(applicationId, application); - queue.getMetrics().submitApp(user); + + if(isAppOnUAM) { + queue.getMetrics().submitUnmanagedApp(user); + } else { + queue.getMetrics().submitApp(user); + } LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName @@ -609,8 +617,11 @@ protected void addApplicationAttempt( } else{ maxRunningEnforcer.trackNonRunnableApp(attempt); } - - queue.getMetrics().submitAppAttempt(user); + if(application.isAppOnUAM()) { + queue.getMetrics().submitUnmanagedAppAttempt(user); + } else { + queue.getMetrics().submitAppAttempt(user); + } LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user: " + user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 9ec4822e513..69df862b268 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -389,11 +389,15 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, @VisibleForTesting public synchronized void addApplication(ApplicationId applicationId, - String queue, String user, boolean isAppRecovering) { + String queue, String user, boolean isAppRecovering, boolean isAppOnUAM) { SchedulerApplication application = - new SchedulerApplication<>(DEFAULT_QUEUE, user); + new SchedulerApplication<>(DEFAULT_QUEUE, user, isAppOnUAM); applications.put(applicationId, application); - metrics.submitApp(user); + if(isAppOnUAM) { + metrics.submitUnmanagedApp(user); + } else { + metrics.submitApp(user); + } LOG.info("Accepted application " + applicationId + " from user: " + user + ", currently num of applications: " + applications.size()); if (isAppRecovering) { @@ -424,7 +428,11 @@ public synchronized void addApplication(ApplicationId applicationId, } application.setCurrentAppAttempt(schedulerApp); - metrics.submitAppAttempt(user); + if(application.isAppOnUAM()) { + metrics.submitUnmanagedAppAttempt(user); + } else { + metrics.submitAppAttempt(user); + } LOG.info("Added Application Attempt " + appAttemptId + " to scheduler from user " + application.getUser()); if (isAttemptRecovering) { @@ -769,7 +777,7 @@ public void handle(SchedulerEvent event) { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; addApplication(appAddedEvent.getApplicationId(), appAddedEvent.getQueue(), appAddedEvent.getUser(), - appAddedEvent.getIsAppRecovering()); + appAddedEvent.getIsAppRecovering(), appAddedEvent.isAppOnUAM()); } break; case APP_REMOVED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index b1a69f019bf..47174a47b05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -405,7 +405,8 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { RMApp mockAPp = new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW); SchedulerApplication application = - new SchedulerApplication(null, mockAPp.getUser()); + new SchedulerApplication(null, mockAPp.getUser(), + false); // Second app with one app attempt RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java index 62e5f3d72ab..391649d4092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java @@ -50,7 +50,7 @@ public void testBacklistChanged() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo(appAttemptId, "test", queue, null, 0, new ResourceUsage(), - new HashMap(), rmContext); + new HashMap(), rmContext, false); appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), new ArrayList()); @@ -124,7 +124,7 @@ public void testSchedulerKeyAccounting() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo info = new AppSchedulingInfo( appAttemptId, "test", queue, mock(ActiveUsersManager.class), 0, - new ResourceUsage(), new HashMap<>(), rmContext); + new ResourceUsage(), new HashMap<>(), rmContext, false); Assert.assertEquals(0, info.getSchedulerKeys().size()); Priority pri1 = Priority.newInstance(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 40650532ed2..7183f7f782f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -796,7 +796,7 @@ public void testNodeUpdateBeforeAppAttemptInit() throws Exception { scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); - scheduler.addApplication(appId, "queue1", "user1", false); + scheduler.addApplication(appId, "queue1", "user1", false, false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try {