diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8e65e6a42e3..a3b235bc3c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -102,11 +102,13 @@ private final Map applicationSchedulingEnvs = new HashMap<>(); private final RMContext rmContext; private final int retryAttempts; + private boolean unmanagedAM; public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, AbstractUsersManager abstractUsersManager, long epoch, ResourceUsage appResourceUsage, - Map applicationSchedulingEnvs, RMContext rmContext) { + Map applicationSchedulingEnvs, RMContext rmContext, + boolean unmanagedAM) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; @@ -120,6 +122,7 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, this.retryAttempts = rmContext.getYarnConfiguration().getInt( YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS); + this.unmanagedAM = unmanagedAM; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); updateContext = new ContainerUpdateContext(this); @@ -156,6 +159,14 @@ public boolean isPending() { return pending; } + public void setUnmanagedAM(boolean unmanagedAM) { + this.unmanagedAM = unmanagedAM; + } + + public boolean isUnmanagedAM() { + return unmanagedAM; + } + public Set getRequestedPartitions() { return requestedPartitions; } @@ -617,8 +628,10 @@ public void move(Queue newQueue) { ap.getPrimaryRequestedNodePartition(), delta); } } - oldMetrics.moveAppFrom(this); - newMetrics.moveAppTo(this); + + oldMetrics.moveAppFrom(this, isUnmanagedAM()); + newMetrics.moveAppTo(this, isUnmanagedAM()); + abstractUsersManager.deactivateApplication(user, applicationId); abstractUsersManager = newQueue.getAbstractUsersManager(); if (!schedulerKeys.isEmpty()) { @@ -649,7 +662,8 @@ public void stop() { ask.getCount())); } } - metrics.finishAppAttempt(applicationId, pending, user); + + metrics.finishAppAttempt(applicationId, pending, user, unmanagedAM); // Clear requests themselves clearRequests(); @@ -695,7 +709,7 @@ public void recoverContainer(RMContainer rmContainer, String partition) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + metrics.runAppAttempt(applicationId, user, isUnmanagedAM()); } // Container is completed. Skip recovering resources. @@ -736,7 +750,7 @@ private void updateMetricsForAllocatedContainer(NodeType type, // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; - metrics.runAppAttempt(applicationId, user); + metrics.runAppAttempt(applicationId, user, isUnmanagedAM()); } updateMetrics(applicationId, type, node, containerAllocated, user, queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 1e42e7a01d9..6315e704c31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -62,6 +62,20 @@ @Metric("# of apps completed") MutableCounterInt appsCompleted; @Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps failed") MutableCounterInt appsFailed; + + @Metric("# of Unmanaged apps submitted") + private MutableCounterInt unmanagedAppsSubmitted; + @Metric("# of Unmanaged running apps") + private MutableGaugeInt unmanagedAppsRunning; + @Metric("# of Unmanaged pending apps") + private MutableGaugeInt unmanagedAppsPending; + @Metric("# of Unmanaged apps completed") + private MutableCounterInt unmanagedAppsCompleted; + @Metric("# of Unmanaged apps killed") + private MutableCounterInt unmanagedAppsKilled; + @Metric("# of Unmanaged apps failed") + private MutableCounterInt unmanagedAppsFailed; + @Metric("Aggregate # of allocated node-local containers") MutableCounterLong aggregateNodeLocalContainersAllocated; @Metric("Aggregate # of allocated rack-local containers") @@ -401,103 +415,158 @@ public void getMetrics(MetricsCollector collector, boolean all) { registry.snapshot(collector.addRecord(registry.info()), all); } - public void submitApp(String user) { + public void submitApp(String user, boolean unmanagedAM) { appsSubmitted.incr(); + if(unmanagedAM) { + unmanagedAppsSubmitted.incr(); + } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.submitApp(user); + userMetrics.submitApp(user, unmanagedAM); } if (parent != null) { - parent.submitApp(user); + parent.submitApp(user, unmanagedAM); } } - public void submitAppAttempt(String user) { + + public void submitAppAttempt(String user, boolean unmanagedAM) { appsPending.incr(); + if(unmanagedAM) { + unmanagedAppsPending.incr(); + } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.submitAppAttempt(user); + userMetrics.submitAppAttempt(user, unmanagedAM); } if (parent != null) { - parent.submitAppAttempt(user); + parent.submitAppAttempt(user, unmanagedAM); } } - public void runAppAttempt(ApplicationId appId, String user) { + public void runAppAttempt(ApplicationId appId, String user, + boolean unmanagedAM) { runBuckets.add(appId, System.currentTimeMillis()); appsRunning.incr(); appsPending.decr(); + + if(unmanagedAM) { + unmanagedAppsRunning.incr(); + unmanagedAppsPending.decr(); + } + QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.runAppAttempt(appId, user); + userMetrics.runAppAttempt(appId, user, unmanagedAM); } if (parent != null) { - parent.runAppAttempt(appId, user); + parent.runAppAttempt(appId, user, unmanagedAM); } } - public void finishAppAttempt( - ApplicationId appId, boolean isPending, String user) { + public void finishAppAttempt(ApplicationId appId, boolean isPending, + String user, boolean unmanagedAM) { runBuckets.remove(appId); if (isPending) { appsPending.decr(); } else { appsRunning.decr(); } + + if(unmanagedAM) { + if (isPending) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + } QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.finishAppAttempt(appId, isPending, user); + userMetrics.finishAppAttempt(appId, isPending, user, unmanagedAM); } if (parent != null) { - parent.finishAppAttempt(appId, isPending, user); + parent.finishAppAttempt(appId, isPending, user, unmanagedAM); } } - public void finishApp(String user, RMAppState rmAppFinalState) { + public void finishApp(String user, RMAppState rmAppFinalState, + boolean unmanagedAM) { switch (rmAppFinalState) { case KILLED: appsKilled.incr(); break; case FAILED: appsFailed.incr(); break; default: appsCompleted.incr(); break; } + + if(unmanagedAM) { + switch (rmAppFinalState) { + case KILLED: + unmanagedAppsKilled.incr(); + break; + case FAILED: + unmanagedAppsFailed.incr(); + break; + default: + unmanagedAppsCompleted.incr(); + break; + } + } + QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { - userMetrics.finishApp(user, rmAppFinalState); + userMetrics.finishApp(user, rmAppFinalState, unmanagedAM); } if (parent != null) { - parent.finishApp(user, rmAppFinalState); + parent.finishApp(user, rmAppFinalState, unmanagedAM); } } - - public void moveAppFrom(AppSchedulingInfo app) { + + + public void moveAppFrom(AppSchedulingInfo app, boolean unmanagedAM) { if (app.isPending()) { appsPending.decr(); } else { appsRunning.decr(); } + if(unmanagedAM) { + if (app.isPending()) { + unmanagedAppsPending.decr(); + } else { + unmanagedAppsRunning.decr(); + } + } + QueueMetrics userMetrics = getUserMetrics(app.getUser()); if (userMetrics != null) { - userMetrics.moveAppFrom(app); + userMetrics.moveAppFrom(app, unmanagedAM); } if (parent != null) { - parent.moveAppFrom(app); + parent.moveAppFrom(app, unmanagedAM); } } - - public void moveAppTo(AppSchedulingInfo app) { + + public void moveAppTo(AppSchedulingInfo app, boolean unmanagedAM) { if (app.isPending()) { appsPending.incr(); } else { appsRunning.incr(); } + if(unmanagedAM) { + if (app.isPending()) { + unmanagedAppsPending.incr(); + } else { + unmanagedAppsRunning.incr(); + } + } QueueMetrics userMetrics = getUserMetrics(app.getUser()); if (userMetrics != null) { - userMetrics.moveAppTo(app); + userMetrics.moveAppTo(app, unmanagedAM); } if (parent != null) { - parent.moveAppTo(app); + parent.moveAppTo(app, unmanagedAM); } } + /** * Set available resources. To be called by scheduler periodically as * resources become available. @@ -1024,18 +1093,34 @@ public int getAppsSubmitted() { return appsSubmitted.value(); } + public int getUnmanagedAppsSubmitted() { + return unmanagedAppsSubmitted.value(); + } + public int getAppsRunning() { return appsRunning.value(); } + public int getUnmanagedAppsRunning() { + return unmanagedAppsRunning.value(); + } + public int getAppsPending() { return appsPending.value(); } + public int getUnmanagedAppsPending() { + return unmanagedAppsPending.value(); + } + public int getAppsCompleted() { return appsCompleted.value(); } + public int getUnmanagedAppsCompleted() { + return unmanagedAppsCompleted.value(); + } + public int getAppsKilled() { return appsKilled.value(); } @@ -1044,6 +1129,10 @@ public int getAppsFailed() { return appsFailed.value(); } + public int getUnmanagedAppsFailed() { + return unmanagedAppsFailed.value(); + } + public Resource getAllocatedResources() { if (queueMetricsForCustomResources != null) { return Resource.newInstance(allocatedMB.value(), allocatedVCores.value(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index 96288f8b9f3..fce7b551c61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -30,16 +30,20 @@ private final String user; private volatile T currentAttempt; private volatile Priority priority; + private boolean unmanagedAM; - public SchedulerApplication(Queue queue, String user) { + public SchedulerApplication(Queue queue, String user, boolean unmanagedAM) { this.queue = queue; this.user = user; + this.unmanagedAM = unmanagedAM; this.priority = null; } - public SchedulerApplication(Queue queue, String user, Priority priority) { + public SchedulerApplication(Queue queue, String user, Priority priority, + boolean unmanagedAM) { this.queue = queue; this.user = user; + this.unmanagedAM = unmanagedAM; this.priority = priority; } @@ -64,7 +68,7 @@ public void setCurrentAppAttempt(T currentAttempt) { } public void stop(RMAppState rmAppFinalState) { - queue.getMetrics().finishApp(user, rmAppFinalState); + queue.getMetrics().finishApp(user, rmAppFinalState, isUnmanagedAM()); } public Priority getPriority() { @@ -80,4 +84,7 @@ public void setPriority(Priority priority) { } } + public boolean isUnmanagedAM() { + return unmanagedAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index e9575b9d5dd..b60aac5ffb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -241,7 +241,7 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, abstractUsersManager, rmContext.getEpoch(), attemptResourceUsage, - applicationSchedulingEnvs, rmContext); + applicationSchedulingEnvs, rmContext, unmanagedAM); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d259c93a1f4..70738c0a5a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -865,7 +865,8 @@ public boolean isAmbiguous(String queueName) { private void addApplicationOnRecovery(ApplicationId applicationId, String queueName, String user, - Priority priority, ApplicationPlacementContext placementContext) { + Priority priority, ApplicationPlacementContext placementContext, + boolean unmanagedAM) { writeLock.lock(); try { //check if the queue needs to be auto-created during recovery @@ -927,9 +928,11 @@ private void addApplicationOnRecovery(ApplicationId applicationId, // Ignore the exception for recovered app as the app was previously // accepted. } - queue.getMetrics().submitApp(user); + queue.getMetrics().submitApp(user, unmanagedAM); + SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + unmanagedAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -1012,7 +1015,7 @@ private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId private void addApplication(ApplicationId applicationId, String queueName, String user, Priority priority, - ApplicationPlacementContext placementContext) { + ApplicationPlacementContext placementContext, boolean unmanagedAM) { writeLock.lock(); try { if (isSystemAppsLimitReached()) { @@ -1116,9 +1119,10 @@ private void addApplication(ApplicationId applicationId, String queueName, return; } // update the metrics - queue.getMetrics().submitApp(user); + queue.getMetrics().submitApp(user, unmanagedAM); SchedulerApplication application = - new SchedulerApplication(queue, user, priority); + new SchedulerApplication(queue, user, priority, + unmanagedAM); applications.put(applicationId, application); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName); @@ -1986,11 +1990,13 @@ public void handle(SchedulerEvent event) { if (!appAddedEvent.getIsAppRecovering()) { addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), + appAddedEvent.isUnmanagedAM()); } else { addApplicationOnRecovery(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(), - appAddedEvent.getPlacementContext()); + appAddedEvent.getPlacementContext(), + appAddedEvent.isUnmanagedAM()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 71e65cb4367..d94aeb081bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -602,7 +602,9 @@ public void submitApplicationAttempt(FiCaSchedulerApp application, // We don't want to update metrics for move app if (!isMoveApp) { - metrics.submitAppAttempt(userName); + boolean unmanagedAM = application.getAppSchedulingInfo() != null && + application.getAppSchedulingInfo().isUnmanagedAM(); + metrics.submitAppAttempt(userName, unmanagedAM); } getParent().submitApplicationAttempt(application, userName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 80b7f2fb821..ea846d058b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -34,6 +34,7 @@ private final boolean isAppRecovering; private final Priority appPriority; private final ApplicationPlacementContext placementContext; + private boolean unmanagedAM = false; public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user) { @@ -58,6 +59,7 @@ public AppAddedSchedulerEvent(String user, this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, null); + this.unmanagedAM = submissionContext.getUnmanagedAM(); } public AppAddedSchedulerEvent(String user, @@ -66,6 +68,7 @@ public AppAddedSchedulerEvent(String user, this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, placementContext); + this.unmanagedAM = submissionContext.getUnmanagedAM(); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, @@ -108,4 +111,8 @@ public Priority getApplicatonPriority() { public ApplicationPlacementContext getPlacementContext() { return placementContext; } + + public boolean isUnmanagedAM() { + return unmanagedAM; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index bc21a17cc73..3790c48ba08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -552,11 +552,15 @@ protected void addApplication(ApplicationId applicationId, return; } } + boolean unmanagedAM = rmApp != null && + rmApp.getApplicationSubmissionContext() != null + && rmApp.getApplicationSubmissionContext().getUnmanagedAM(); SchedulerApplication application = - new SchedulerApplication<>(queue, user); + new SchedulerApplication<>(queue, user, unmanagedAM); applications.put(applicationId, application); - queue.getMetrics().submitApp(user); + + queue.getMetrics().submitApp(user, unmanagedAM); LOG.info("Accepted application " + applicationId + " from user: " + user + ", in queue: " + queueName @@ -610,7 +614,7 @@ protected void addApplicationAttempt( maxRunningEnforcer.trackNonRunnableApp(attempt); } - queue.getMetrics().submitAppAttempt(user); + queue.getMetrics().submitAppAttempt(user, application.isUnmanagedAM()); LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user: " + user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 9ec4822e513..c1a591eba0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -389,11 +389,13 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, @VisibleForTesting public synchronized void addApplication(ApplicationId applicationId, - String queue, String user, boolean isAppRecovering) { + String queue, String user, boolean isAppRecovering, + boolean unmanagedAM) { SchedulerApplication application = - new SchedulerApplication<>(DEFAULT_QUEUE, user); + new SchedulerApplication<>(DEFAULT_QUEUE, user, unmanagedAM); applications.put(applicationId, application); - metrics.submitApp(user); + + metrics.submitApp(user, unmanagedAM); LOG.info("Accepted application " + applicationId + " from user: " + user + ", currently num of applications: " + applications.size()); if (isAppRecovering) { @@ -424,7 +426,8 @@ public synchronized void addApplication(ApplicationId applicationId, } application.setCurrentAppAttempt(schedulerApp); - metrics.submitAppAttempt(user); + metrics.submitAppAttempt(user, application.isUnmanagedAM()); + LOG.info("Added Application Attempt " + appAttemptId + " to scheduler from user " + application.getUser()); if (isAttemptRecovering) { @@ -768,8 +771,8 @@ public void handle(SchedulerEvent event) { { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; addApplication(appAddedEvent.getApplicationId(), - appAddedEvent.getQueue(), appAddedEvent.getUser(), - appAddedEvent.getIsAppRecovering()); + appAddedEvent.getQueue(), appAddedEvent.getUser(), + appAddedEvent.getIsAppRecovering(), appAddedEvent.isUnmanagedAM()); } break; case APP_REMOVED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index da7f26bea3f..c5f45fd9789 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1559,6 +1559,8 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); + QueueMetrics qm1 = rm1.getResourceScheduler().getRootQueueMetrics(); + assertUnmanagedAMQueueMetrics(qm1, 0, 0, 0, 0); // create app and launch the UAM MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithMemory(200, rm1) @@ -1567,6 +1569,7 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { RMApp app0 = MockRMAppSubmitter.submit(rm1, data); MockAM am0 = MockRM.launchUAM(app0, rm1, nm1); am0.registerAppAttempt(); + assertUnmanagedAMQueueMetrics(qm1, 1, 1, 0, 0); // Allocate containers to UAM int numContainers = 2; @@ -1581,17 +1584,19 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { new ArrayList()).getAllocatedContainers()); Thread.sleep(100); } + assertUnmanagedAMQueueMetrics(qm1, 1, 0, 1, 0); // start new RM rm2 = new MockRM(conf, memStore); rm2.start(); MockMemoryRMStateStore memStore2 = (MockMemoryRMStateStore) rm2.getRMStateStore(); + QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics(); rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED); - // recover app nm1.setResourceTrackerService(rm2.getResourceTrackerService()); + assertUnmanagedAMQueueMetrics(qm2, 1, 1, 0, 0); RMApp recoveredApp = rm2.getRMContext().getRMApps().get(app0.getApplicationId()); NMContainerStatus container1 = TestRMRestart @@ -1601,13 +1606,13 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { .createNMContainerStatus(am0.getApplicationAttemptId(), 2, ContainerState.RUNNING); nm1.registerNode(Arrays.asList(container1, container2), null); - // Wait for RM to settle down on recovering containers; waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId()); // retry registerApplicationMaster() after RM restart. am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); am0.registerAppAttempt(true); + assertUnmanagedAMQueueMetrics(qm2, 1, 0, 1, 0); // Check if UAM is correctly recovered on restart rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING); @@ -1626,6 +1631,7 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { // Check if UAM is able to heart beat Assert.assertNotNull(am0.doHeartbeat()); + assertUnmanagedAMQueueMetrics(qm2, 1, 0, 1, 0); // Complete the UAM am0.unregisterAppAttempt(false); @@ -1633,15 +1639,26 @@ public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception { rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED); Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, recoveredApp.getFinalApplicationStatus()); + assertUnmanagedAMQueueMetrics(qm2, 1, 0, 0, 1); // Restart RM once more to check UAM is not re-run MockRM rm3 = new MockRM(conf, memStore2); rm3.start(); recoveredApp = rm3.getRMContext().getRMApps().get(app0.getApplicationId()); + QueueMetrics qm3 = rm3.getResourceScheduler().getRootQueueMetrics(); Assert.assertEquals(RMAppState.FINISHED, recoveredApp.getState()); + assertUnmanagedAMQueueMetrics(qm2, 1, 0, 0, 1); + } + private void assertUnmanagedAMQueueMetrics(QueueMetrics qm, int appsSubmitted, + int appsPending, int appsRunning, int appsCompleted) { + Assert.assertEquals(appsSubmitted, qm.getUnmanagedAppsSubmitted()); + Assert.assertEquals(appsPending, qm.getUnmanagedAppsPending()); + Assert.assertEquals(appsRunning, qm.getUnmanagedAppsRunning()); + Assert.assertEquals(appsCompleted, qm.getUnmanagedAppsCompleted()); } + @Test(timeout = 30000) public void testUnknownUserOnRecovery() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java index b9506be4b3e..19ee08f17cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppMetricsChecker.java @@ -33,6 +33,12 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_PENDING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_RUNNING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_SUBMITTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_COMPLETED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_FAILED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_KILLED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_PENDING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_RUNNING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_SUBMITTED; final class AppMetricsChecker { private final static Logger LOG = @@ -45,7 +51,13 @@ .gaugeInt(APPS_RUNNING, 0) .counter(APPS_COMPLETED, 0) .counter(APPS_FAILED, 0) - .counter(APPS_KILLED, 0); + .counter(APPS_KILLED, 0) + .counter(UNMANAGED_APPS_SUBMITTED, 0) + .gaugeInt(UNMANAGED_APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0) + .counter(UNMANAGED_APPS_COMPLETED, 0) + .counter(UNMANAGED_APPS_FAILED, 0) + .counter(UNMANAGED_APPS_KILLED, 0); enum AppMetricsKey { APPS_SUBMITTED("AppsSubmitted"), @@ -53,7 +65,13 @@ APPS_RUNNING("AppsRunning"), APPS_COMPLETED("AppsCompleted"), APPS_FAILED("AppsFailed"), - APPS_KILLED("AppsKilled"); + APPS_KILLED("AppsKilled"), + UNMANAGED_APPS_SUBMITTED("UnmanagedAppsSubmitted"), + UNMANAGED_APPS_PENDING("UnmanagedAppsPending"), + UNMANAGED_APPS_RUNNING("UnmanagedAppsRunning"), + UNMANAGED_APPS_COMPLETED("UnmanagedAppsCompleted"), + UNMANAGED_APPS_FAILED("UnmanagedAppsFailed"), + UNMANAGED_APPS_KILLED("UnmanagedAppsKilled"); private String value; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index b1a69f019bf..47174a47b05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -405,7 +405,8 @@ public void testReleasedContainerIfAppAttemptisNull() throws Exception { RMApp mockAPp = new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW); SchedulerApplication application = - new SchedulerApplication(null, mockAPp.getUser()); + new SchedulerApplication(null, mockAPp.getUser(), + false); // Second app with one app attempt RMApp app = MockRMAppSubmitter.submitWithMemory(200, rm1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java index 62e5f3d72ab..391649d4092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java @@ -50,7 +50,7 @@ public void testBacklistChanged() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo appSchedulingInfo = new AppSchedulingInfo(appAttemptId, "test", queue, null, 0, new ResourceUsage(), - new HashMap(), rmContext); + new HashMap(), rmContext, false); appSchedulingInfo.updatePlacesBlacklistedByApp(new ArrayList(), new ArrayList()); @@ -124,7 +124,7 @@ public void testSchedulerKeyAccounting() { doReturn(new YarnConfiguration()).when(rmContext).getYarnConfiguration(); AppSchedulingInfo info = new AppSchedulingInfo( appAttemptId, "test", queue, mock(ActiveUsersManager.class), 0, - new ResourceUsage(), new HashMap<>(), rmContext); + new ResourceUsage(), new HashMap<>(), rmContext, false); Assert.assertEquals(0, info.getSchedulerKeys().size()); Priority pri1 = Priority.newInstance(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestPartitionQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestPartitionQueueMetrics.java index eb240d1b6d3..ab4296cc4fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestPartitionQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestPartitionQueueMetrics.java @@ -86,8 +86,8 @@ public void testSinglePartitionWithSingleLevelQueueMetrics() QueueMetrics q2 = QueueMetrics.forQueue(ms, "root.q2", parentQueue, true, CONF); - q1.submitApp(user); - q1.submitAppAttempt(user); + q1.submitApp(user, false); + q1.submitAppAttempt(user, false); root.setAvailableResourcesToQueue("x", Resources.createResource(200 * GB, 200)); @@ -140,8 +140,8 @@ public void testTwoPartitionWithSingleLevelQueueMetrics() throws Exception { QueueMetrics.forQueue(ms, "root.q2", parentQueue, false, CONF); AppSchedulingInfo app = mockApp(user); - q1.submitApp(user); - q1.submitAppAttempt(user); + q1.submitApp(user, false); + q1.submitAppAttempt(user, false); root.setAvailableResourcesToQueue("x", Resources.createResource(200 * GB, 200)); @@ -414,8 +414,8 @@ public void testTwoLevelWithUserMetrics() { QueueMetrics.forQueue(ms, leafQueueName, parentQueue, true, CONF); AppSchedulingInfo app = mockApp(user); - metrics.submitApp(user); - metrics.submitAppAttempt(user); + metrics.submitApp(user, false); + metrics.submitAppAttempt(user, false); parentMetrics.setAvailableResourcesToQueue(partition, Resources.createResource(100 * GB, 100)); @@ -447,7 +447,7 @@ public void testTwoLevelWithUserMetrics() { checkResources(partitionSource, 0, 0, 0, 0, 0, 100 * GB, 100, 18 * GB, 18, 6, 0, 0, 0); - metrics.runAppAttempt(app.getApplicationId(), user); + metrics.runAppAttempt(app.getApplicationId(), user, false); metrics.allocateResources(partition, user, 3, Resources.createResource(1 * GB, 1), true); @@ -491,9 +491,9 @@ public void testTwoLevelWithUserMetrics() { 0, 0, 0); metrics.finishAppAttempt(app.getApplicationId(), app.isPending(), - app.getUser()); + app.getUser(), false); - metrics.finishApp(user, RMAppState.FINISHED); + metrics.finishApp(user, RMAppState.FINISHED, false); } @Test @@ -519,8 +519,8 @@ public void testThreeLevelWithUserMetrics() { QueueMetrics.forQueue(leafQueueName1, leafQueue, true, CONF); AppSchedulingInfo app = mockApp(user); - metrics1.submitApp(user); - metrics1.submitAppAttempt(user); + metrics1.submitApp(user, false); + metrics1.submitAppAttempt(user, false); parentMetrics.setAvailableResourcesToQueue(partitionX, Resources.createResource(200 * GB, 200)); @@ -615,9 +615,9 @@ public void testThreeLevelWithUserMetrics() { 0, 0); metrics1.finishAppAttempt(app.getApplicationId(), app.isPending(), - app.getUser()); + app.getUser(), false); - metrics1.finishApp(user, RMAppState.FINISHED); + metrics1.finishApp(user, RMAppState.FINISHED, false); } /** @@ -650,8 +650,8 @@ public void testSinglePartitionWithSingleLevelQueueMetricsWithoutUserMetrics() AppSchedulingInfo app = mockApp(user); - q1.submitApp(user); - q1.submitAppAttempt(user); + q1.submitApp(user, false); + q1.submitAppAttempt(user, false); root.setAvailableResourcesToQueue("x", Resources.createResource(200 * GB, 200)); @@ -680,8 +680,9 @@ public void testSinglePartitionWithSingleLevelQueueMetricsWithoutUserMetrics() checkResources(q2Source, 0, 0, 0, 0, 0, 3 * GB, 3, 3); checkResources(q2UserSource, 0, 0, 0, 0, 0, 3 * GB, 3, 3); - q1.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser()); - q1.finishApp(user, RMAppState.FINISHED); + q1.finishAppAttempt(app.getApplicationId(), app.isPending(), app.getUser(), + false); + q1.finishApp(user, RMAppState.FINISHED, false); } public static MetricsSource partitionSource(MetricsSystem ms, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 33c39290de5..2137285bac0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -44,6 +44,10 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_PENDING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_RUNNING; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.APPS_SUBMITTED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_FAILED; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_PENDING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_RUNNING; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppMetricsChecker.AppMetricsKey.UNMANAGED_APPS_SUBMITTED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_ALLOCATED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.AGGREGATE_CONTAINERS_RELEASED; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceMetricsChecker.ResourceMetricsKey.ALLOCATED_CONTAINERS; @@ -89,12 +93,12 @@ public void testDefaultSingleQueueMetrics() { MetricsSource queueSource= queueSource(ms, queueName); AppSchedulingInfo app = mockApp(USER); - metrics.submitApp(USER); + metrics.submitApp(USER, false); MetricsSource userSource = userSource(ms, queueName, USER); AppMetricsChecker appMetricsChecker = AppMetricsChecker.create() .counter(APPS_SUBMITTED, 1) .checkAgainst(queueSource, true); - metrics.submitAppAttempt(USER); + metrics.submitAppAttempt(USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 1) .checkAgainst(queueSource, true); @@ -111,7 +115,7 @@ public void testDefaultSingleQueueMetrics() { .gaugeLong(PENDING_MB, 15 * GB).gaugeInt(PENDING_V_CORES, 15) .gaugeInt(PENDING_CONTAINERS, 5).checkAgainst(queueSource); - metrics.runAppAttempt(app.getApplicationId(), USER); + metrics.runAppAttempt(app.getApplicationId(), USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 0) .gaugeInt(APPS_RUNNING, 1) @@ -151,12 +155,12 @@ public void testDefaultSingleQueueMetrics() { .checkAgainst(queueSource); metrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .counter(APPS_SUBMITTED, 1) .gaugeInt(APPS_RUNNING, 0) .checkAgainst(queueSource, true); - metrics.finishApp(USER, RMAppState.FINISHED); + metrics.finishApp(USER, RMAppState.FINISHED, false); AppMetricsChecker.createFromChecker(appMetricsChecker) .counter(APPS_COMPLETED, 1) .checkAgainst(queueSource, true); @@ -172,36 +176,36 @@ public void testQueueAppMetricsForMultipleFailures() { MetricsSource queueSource = queueSource(ms, queueName); AppSchedulingInfo app = mockApp(USER); - metrics.submitApp(USER); + metrics.submitApp(USER, false); MetricsSource userSource = userSource(ms, queueName, USER); AppMetricsChecker appMetricsChecker = AppMetricsChecker.create() .counter(APPS_SUBMITTED, 1) .checkAgainst(queueSource, true); - metrics.submitAppAttempt(USER); + metrics.submitAppAttempt(USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 1) .checkAgainst(queueSource, true); - metrics.runAppAttempt(app.getApplicationId(), USER); + metrics.runAppAttempt(app.getApplicationId(), USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 0) .gaugeInt(APPS_RUNNING, 1) .checkAgainst(queueSource, true); metrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_RUNNING, 0) .checkAgainst(queueSource, true); // As the application has failed, framework retries the same application // based on configuration - metrics.submitAppAttempt(USER); + metrics.submitAppAttempt(USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 1) .checkAgainst(queueSource, true); - metrics.runAppAttempt(app.getApplicationId(), USER); + metrics.runAppAttempt(app.getApplicationId(), USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 0) .gaugeInt(APPS_RUNNING, 1) @@ -209,19 +213,19 @@ public void testQueueAppMetricsForMultipleFailures() { // Suppose say application has failed this time as well. metrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_RUNNING, 0) .checkAgainst(queueSource, true); // As the application has failed, framework retries the same application // based on configuration - metrics.submitAppAttempt(USER); + metrics.submitAppAttempt(USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 1) .checkAgainst(queueSource, true); - metrics.runAppAttempt(app.getApplicationId(), USER); + metrics.runAppAttempt(app.getApplicationId(), USER, false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_PENDING, 0) .gaugeInt(APPS_RUNNING, 1) @@ -229,12 +233,12 @@ public void testQueueAppMetricsForMultipleFailures() { // Suppose say application has failed, and there's no more retries. metrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_RUNNING, 0) .checkAgainst(queueSource, true); - metrics.finishApp(USER, RMAppState.FAILED); + metrics.finishApp(USER, RMAppState.FAILED, false); AppMetricsChecker.createFromChecker(appMetricsChecker) .gaugeInt(APPS_RUNNING, 0) .counter(APPS_FAILED, 1) @@ -243,6 +247,87 @@ public void testQueueAppMetricsForMultipleFailures() { assertNull(userSource); } + @Test + public void testQueueUnmanagedAppMetricsForMultipleFailures() { + String queueName = "single"; + + QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, + new Configuration()); + MetricsSource queueSource = queueSource(ms, queueName); + AppSchedulingInfo app = mockApp(USER); + + // Submit an unmanaged Application. + metrics.submitApp(USER, true); + MetricsSource userSource = userSource(ms, queueName, USER); + AppMetricsChecker appMetricsChecker = AppMetricsChecker.create() + .counter(UNMANAGED_APPS_SUBMITTED, 1).counter(APPS_SUBMITTED, 1) + .checkAgainst(queueSource, true); + metrics.submitAppAttempt(USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1).gaugeInt(APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runAppAttempt(app.getApplicationId(), USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0).gaugeInt(APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1).gaugeInt(APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + metrics.finishAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser(), true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0).gaugeInt(APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + // As the application has failed, framework retries the same application + // based on configuration + metrics.submitAppAttempt(USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1).gaugeInt(APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runAppAttempt(app.getApplicationId(), USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0).gaugeInt(APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1).gaugeInt(APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + // Suppose say application has failed this time as well. + metrics.finishAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser(), true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0).gaugeInt(APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + // As the application has failed, framework retries the same application + // based on configuration + metrics.submitAppAttempt(USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 1).gaugeInt(APPS_PENDING, 1) + .checkAgainst(queueSource, true); + + metrics.runAppAttempt(app.getApplicationId(), USER, true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_PENDING, 0).gaugeInt(APPS_PENDING, 0) + .gaugeInt(UNMANAGED_APPS_RUNNING, 1).gaugeInt(APPS_RUNNING, 1) + .checkAgainst(queueSource, true); + + // Suppose say application has failed, and there's no more retries. + metrics.finishAppAttempt( + app.getApplicationId(), app.isPending(), app.getUser(), true); + appMetricsChecker = AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0).gaugeInt(APPS_RUNNING, 0) + .checkAgainst(queueSource, true); + + metrics.finishApp(USER, RMAppState.FAILED, true); + AppMetricsChecker.createFromChecker(appMetricsChecker) + .gaugeInt(UNMANAGED_APPS_RUNNING, 0).gaugeInt(APPS_RUNNING, 0) + .counter(UNMANAGED_APPS_FAILED, 1).counter(APPS_FAILED, 1) + .checkAgainst(queueSource, true); + + assertNull(userSource); + } + @Test public void testSingleQueueWithUserMetrics() { String queueName = "single2"; @@ -252,7 +337,7 @@ public void testSingleQueueWithUserMetrics() { MetricsSource queueSource = queueSource(ms, queueName); AppSchedulingInfo app = mockApp(USER_2); - metrics.submitApp(USER_2); + metrics.submitApp(USER_2, false); MetricsSource userSource = userSource(ms, queueName, USER_2); AppMetricsChecker appMetricsQueueSourceChecker = AppMetricsChecker.create() @@ -262,7 +347,7 @@ public void testSingleQueueWithUserMetrics() { .counter(APPS_SUBMITTED, 1) .checkAgainst(userSource, true); - metrics.submitAppAttempt(USER_2); + metrics.submitAppAttempt(USER_2, false); appMetricsQueueSourceChecker = AppMetricsChecker .createFromChecker(appMetricsQueueSourceChecker) .gaugeInt(APPS_PENDING, 1) @@ -298,7 +383,7 @@ public void testSingleQueueWithUserMetrics() { .gaugeInt(PENDING_CONTAINERS, 5) .checkAgainst(userSource); - metrics.runAppAttempt(app.getApplicationId(), USER_2); + metrics.runAppAttempt(app.getApplicationId(), USER_2, false); appMetricsQueueSourceChecker = AppMetricsChecker .createFromChecker(appMetricsQueueSourceChecker) .gaugeInt(APPS_PENDING, 0) @@ -349,7 +434,7 @@ public void testSingleQueueWithUserMetrics() { .checkAgainst(userSource); metrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsQueueSourceChecker = AppMetricsChecker.createFromChecker(appMetricsQueueSourceChecker) .gaugeInt(APPS_RUNNING, 0) @@ -358,7 +443,7 @@ public void testSingleQueueWithUserMetrics() { AppMetricsChecker.createFromChecker(appMetricsUserSourceChecker) .gaugeInt(APPS_RUNNING, 0) .checkAgainst(userSource, true); - metrics.finishApp(USER_2, RMAppState.FINISHED); + metrics.finishApp(USER_2, RMAppState.FINISHED, false); AppMetricsChecker.createFromChecker(appMetricsQueueSourceChecker) .counter(APPS_COMPLETED, 1) .checkAgainst(queueSource, true); @@ -382,7 +467,7 @@ public void testNodeTypeMetrics() { MetricsSource queueSource = queueSource(ms, leafQueueName); //AppSchedulingInfo app = mockApp(user); - metrics.submitApp(USER); + metrics.submitApp(USER, false); MetricsSource userSource = userSource(ms, leafQueueName, USER); MetricsSource parentUserSource = userSource(ms, parentQueueName, USER); @@ -417,7 +502,7 @@ public void testTwoLevelWithUserMetrics() { QueueInfo root = new QueueInfo(null, "root", ms, conf, USER); QueueInfo leaf = new QueueInfo(root, "root.leaf", ms, conf, USER); - leaf.queueMetrics.submitApp(USER); + leaf.queueMetrics.submitApp(USER, false); AppMetricsChecker appMetricsQueueSourceChecker = AppMetricsChecker.create() .counter(APPS_SUBMITTED, 1) @@ -434,7 +519,7 @@ public void testTwoLevelWithUserMetrics() { .counter(APPS_SUBMITTED, 1) .checkAgainst(root.userSource, true); - leaf.queueMetrics.submitAppAttempt(USER); + leaf.queueMetrics.submitAppAttempt(USER, false); appMetricsQueueSourceChecker = AppMetricsChecker.createFromChecker(appMetricsQueueSourceChecker) .gaugeInt(APPS_PENDING, 1) @@ -489,7 +574,7 @@ public void testTwoLevelWithUserMetrics() { .gaugeLong(PENDING_MB, 15 * GB).gaugeInt(PENDING_V_CORES, 15) .gaugeInt(PENDING_CONTAINERS, 5).checkAgainst(root.userSource); - leaf.queueMetrics.runAppAttempt(app.getApplicationId(), USER); + leaf.queueMetrics.runAppAttempt(app.getApplicationId(), USER, false); appMetricsQueueSourceChecker = AppMetricsChecker.createFromChecker(appMetricsQueueSourceChecker) .gaugeInt(APPS_PENDING, 0) @@ -603,7 +688,7 @@ public void testTwoLevelWithUserMetrics() { .checkAgainst(root.userSource); leaf.queueMetrics.finishAppAttempt( - app.getApplicationId(), app.isPending(), app.getUser()); + app.getApplicationId(), app.isPending(), app.getUser(), false); appMetricsQueueSourceChecker = AppMetricsChecker .createFromChecker(appMetricsQueueSourceChecker) .counter(APPS_SUBMITTED, 1) @@ -627,7 +712,7 @@ public void testTwoLevelWithUserMetrics() { .gaugeInt(APPS_RUNNING, 0) .checkAgainst(root.userSource, true); - leaf.queueMetrics.finishApp(USER, RMAppState.FINISHED); + leaf.queueMetrics.finishApp(USER, RMAppState.FINISHED, false); AppMetricsChecker.createFromChecker(appMetricsQueueSourceChecker) .counter(APPS_COMPLETED, 1) .checkAgainst(leaf.queueSource, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 2a8f57956ba..65eed96c925 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -126,7 +126,8 @@ public void testMove() { when(rmContext.getYarnConfiguration()).thenReturn(conf); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.getAbstractUsersManager(), rmContext); - oldMetrics.submitApp(user); + app.appSchedulingInfo.setUnmanagedAM(false); + oldMetrics.submitApp(user, false); // confirm that containerId is calculated based on epoch. assertEquals(0x30000000001L, app.getNewContainerId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4b023107b39..a879c363540 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -60,6 +60,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -131,6 +133,7 @@ CapacityScheduler cs; CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + private RMApp rmApp; CSQueue root; private CSQueueStore queues; @@ -174,7 +177,7 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) ConcurrentMap spyApps = spy(new ConcurrentHashMap()); - RMApp rmApp = mock(RMApp.class); + rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt(any())).thenReturn(null); amResourceRequest = mock(ResourceRequest.class); when(amResourceRequest.getCapability()).thenReturn( @@ -466,7 +469,12 @@ public void testPolicyConfiguration() throws Exception { public void testAppAttemptMetrics() throws Exception { CSMaxRunningAppsEnforcer enforcer = mock(CSMaxRunningAppsEnforcer.class); cs.setMaxRunningAppsEnforcer(enforcer); - + ApplicationSubmissionContext applicationSubmissionContext = + mock(ApplicationSubmissionContext.class); + when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(false); + when(rmApp.getApplicationSubmissionContext()) + .thenReturn(applicationSubmissionContext); + when(rmApp.getCurrentAppAttempt()).thenReturn(mock(RMAppAttempt.class)); // Manipulate queue 'a' LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); @@ -495,17 +503,18 @@ public void testAppAttemptMetrics() throws Exception { // Attempt the same application again final ApplicationAttemptId appAttemptId_1 = TestUtils .getMockApplicationAttemptId(0, 2); - FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null, - spyRMContext); - app_1.setAMResource(Resource.newInstance(100, 1)); - a.submitApplicationAttempt(app_1, user_0); // same user + FiCaSchedulerApp app1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, + null, spyRMContext); + app1.getAppSchedulingInfo().setUnmanagedAM(false); + app1.setAMResource(Resource.newInstance(100, 1)); + a.submitApplicationAttempt(app1, user_0); // same user assertEquals(1, a.getMetrics().getAppsSubmitted()); assertEquals(1, a.getMetrics().getAppsPending()); assertEquals(1, a.getUser(user_0).getActiveApplications()); - assertEquals(app_1.getAMResource().getMemorySize(), a.getMetrics() + assertEquals(app1.getAMResource().getMemorySize(), a.getMetrics() .getUsedAMResourceMB()); - assertEquals(app_1.getAMResource().getVirtualCores(), a.getMetrics() + assertEquals(app1.getAMResource().getVirtualCores(), a.getMetrics() .getUsedAMResourceVCores()); event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, @@ -524,6 +533,74 @@ public void testAppAttemptMetrics() throws Exception { assertEquals(1, userMetrics.getAppsSubmitted()); } + @Test + public void testUnmanagedAppAttemptMetrics() throws Exception { + CSMaxRunningAppsEnforcer enforcer = mock(CSMaxRunningAppsEnforcer.class); + cs.setMaxRunningAppsEnforcer(enforcer); + // Manipulate queue 'a' + LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); + + // Users + final String user0 = "user_0"; + + // Submit applications + final ApplicationAttemptId appAttemptId0 = TestUtils + .getMockApplicationAttemptId(0, 1); + + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + appAttemptId0.getApplicationId(), "test", a.getQueuePath(), + Priority.newInstance(0), null, true, true, + 2, null, "test"); + + AppAddedSchedulerEvent addAppEvent = + new AppAddedSchedulerEvent(user0, applicationSubmissionContext, false, + null); + cs.handle(addAppEvent); + AppAttemptAddedSchedulerEvent addAttemptEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId0, false); + cs.handle(addAttemptEvent); + + AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent( + appAttemptId0, RMAppAttemptState.FAILED, false); + cs.handle(event); + + assertEquals(0, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(0, a.getMetrics().getUnmanagedAppsFailed()); + + // Attempt the same application again + final ApplicationAttemptId appAttemptId1 = TestUtils + .getMockApplicationAttemptId(0, 2); + FiCaSchedulerApp app1 = new FiCaSchedulerApp(appAttemptId1, user0, a, + null, spyRMContext); + + app1.setAMResource(Resource.newInstance(100, 1)); + a.submitApplicationAttempt(app1, user0); // same user + + assertEquals(1, a.getMetrics().getUnmanagedAppsSubmitted()); + assertEquals(1, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(1, a.getUser(user0).getActiveApplications()); + assertEquals(app1.getAMResource().getMemorySize(), a.getMetrics() + .getUsedAMResourceMB()); + assertEquals(app1.getAMResource().getVirtualCores(), a.getMetrics() + .getUsedAMResourceVCores()); + + event = new AppAttemptRemovedSchedulerEvent(appAttemptId0, + RMAppAttemptState.FINISHED, false); + cs.handle(event); + AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent( + appAttemptId0.getApplicationId(), RMAppState.FINISHED); + cs.handle(rEvent); + + assertEquals(1, a.getMetrics().getUnmanagedAppsSubmitted()); + assertEquals(0, a.getMetrics().getUnmanagedAppsPending()); + assertEquals(0, a.getMetrics().getUnmanagedAppsFailed()); + assertEquals(1, a.getMetrics().getUnmanagedAppsCompleted()); + + QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user0); + assertEquals(1, userMetrics.getUnmanagedAppsSubmitted()); + } + @Test public void testFairConfiguration() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 40650532ed2..7183f7f782f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -796,7 +796,7 @@ public void testNodeUpdateBeforeAppAttemptInit() throws Exception { scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); - scheduler.addApplication(appId, "queue1", "user1", false); + scheduler.addApplication(appId, "queue1", "user1", false, false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try {