diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 507b798a56280ed19bcdc6a2fcda254f5114dd37..67c17a949d61acd880a9f265f82e26703af5e14b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -38,6 +38,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterInt; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -74,6 +75,7 @@ @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; + @Metric("Container First Attempt Allocation Delay") MutableRate containerFirstAttemptAllocationDelay; private final MutableGaugeInt[] runningTime; private TimeBucketMetrics runBuckets; @@ -462,6 +464,10 @@ public void deactivateApp(String user) { parent.deactivateApp(user); } } + + public void addContainerFirstAttemptAllocationDelay(long latency) { + containerFirstAttemptAllocationDelay.add(latency); + } public int getAppsSubmitted() { return appsSubmitted.value(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index d5b6ce669873646173fb4501594e10db677b4698..558eac5e20e1fc763a148fb621faceccb8b70c5c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; @@ -93,6 +94,8 @@ private boolean unmanagedAM = true; private boolean amRunning = false; private LogAggregationContext logAggregationContext; + private AtomicLong firstAllocationRequestSentTime = new AtomicLong(0); + private AtomicLong firstContainerAllocatedTime = new AtomicLong(0); protected List newlyAllocatedContainers = new ArrayList(); @@ -246,6 +249,22 @@ public synchronized int getReReservations(Priority priority) { return reReservations.count(priority); } + public boolean trySetFirstAllocationRequestSentTime(long value) { + return firstAllocationRequestSentTime.compareAndSet(0,value); + } + + public long getFirstAllocationRequestSentTime() { + return firstAllocationRequestSentTime.longValue(); + } + + public boolean trySetFirstContainerAllocatedTime(long value) { + return firstContainerAllocatedTime.compareAndSet(0,value); + } + + public long getFirstContainerAllocatedTime() { + return firstContainerAllocatedTime.longValue(); + } + /** * Get total current reservations. * Used only by unit tests diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1ace6040b25bb512575ac8863b780ddddbac505a..0d36baac4573a37659a8c969bab837259a17a874 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; @@ -896,6 +897,10 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, clusterResource, minimumAllocation, getMaximumResourceCapability(), incrAllocation); + // Record container allocation start time + long start = getClock().getTime(); + application.trySetFirstAllocationRequestSentTime(start); + // Set amResource for this app if (!application.getUnmanagedAM() && ask.size() == 1 && application.getLiveContainers().isEmpty()) { @@ -938,6 +943,28 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, application.updateBlacklist(blacklistAdditions, blacklistRemovals); ContainersAndNMTokensAllocation allocation = application.pullNewlyAllocatedContainersAndNMTokens(); + + // Calculate container allocation latency + if (!(allocation.getContainerList().isEmpty())) { + long firstContainerAllocatedTime = getClock().getTime(); + boolean changed = application + .trySetFirstContainerAllocatedTime(firstContainerAllocatedTime); + if (changed) { + long timediff = application.getFirstContainerAllocatedTime() + - application.getFirstAllocationRequestSentTime(); + if (timediff>0) { + RMApp rmApp = rmContext.getRMApps().get(appAttemptId.getApplicationId()); + if (rmApp!=null) { + String metricsQueue = rmApp.getQueue(); + if (metricsQueue!=null) { + FSQueue queue = queueMgr.getQueue(metricsQueue); + queue.getMetrics().addContainerFirstAttemptAllocationDelay(timediff); + } + } + } + } + } + return new Allocation(allocation.getContainerList(), application.getHeadroom(), preemptionContainerIds, null, null, allocation.getNMTokenList());