diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 507b798a56280ed19bcdc6a2fcda254f5114dd37..58b1ed11513744072a25115c9b16acde10b9ec39 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -38,6 +38,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterInt; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -74,6 +75,7 @@ @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; + @Metric("App Attempt First Container Allocation Delay") MutableRate appAttemptFirstContainerAllocationDelay; private final MutableGaugeInt[] runningTime; private TimeBucketMetrics runBuckets; @@ -462,7 +464,11 @@ public void deactivateApp(String user) { parent.deactivateApp(user); } } - + + public void addAppAttemptFirstContainerAllocationDelay(long latency) { + appAttemptFirstContainerAllocationDelay.add(latency); + } + public int getAppsSubmitted() { return appsSubmitted.value(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 9816699851c47a78c48249d3e3810816209f106c..2c1c911269695218637683aed6af811616608b2d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; @@ -94,6 +95,8 @@ private boolean unmanagedAM = true; private boolean amRunning = false; private LogAggregationContext logAggregationContext; + private AtomicLong firstAllocationRequestSentTime = new AtomicLong(0); + private AtomicLong firstContainerAllocatedTime = new AtomicLong(0); protected List newlyAllocatedContainers = new ArrayList(); @@ -642,4 +645,18 @@ public void setApplicationHeadroomForMetrics(Resource headroom) { Resources.clone(headroom)); } } + + public void recordContainerRequestTime(long value) { + firstAllocationRequestSentTime.compareAndSet(0, value); + } + + public void recordContainerAllocationTime(long value) { + if (firstContainerAllocatedTime.compareAndSet(0, value)) { + long timediff = firstContainerAllocatedTime.longValue() - + firstAllocationRequestSentTime.longValue(); + if (timediff > 0) { + queue.getMetrics().addAppAttemptFirstContainerAllocationDelay(timediff); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 1d97983778d3a5c8bfa398c3570b641aef12f229..9501b62de35421d031cf75e7562a446e82b924b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -896,6 +896,9 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, clusterResource, minimumAllocation, getMaximumResourceCapability(), incrAllocation); + // Record container allocation start time + application.recordContainerRequestTime(getClock().getTime()); + // Set amResource for this app if (!application.getUnmanagedAM() && ask.size() == 1 && application.getLiveContainers().isEmpty()) { @@ -929,7 +932,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, LOG.debug("Preempting " + application.getPreemptionContainers().size() + " container(s)"); } - + Set preemptionContainerIds = new HashSet(); for (RMContainer container : application.getPreemptionContainers()) { preemptionContainerIds.add(container.getContainerId()); @@ -938,6 +941,12 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, application.updateBlacklist(blacklistAdditions, blacklistRemovals); ContainersAndNMTokensAllocation allocation = application.pullNewlyAllocatedContainersAndNMTokens(); + + // Record container allocation time + if (!(allocation.getContainerList().isEmpty())) { + application.recordContainerAllocationTime(getClock().getTime()); + } + Resource headroom = application.getHeadroom(); application.setApplicationHeadroomForMetrics(headroom); return new Allocation(allocation.getContainerList(), headroom,