diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index d2e81a50d94..9d2b0586dc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -877,6 +877,16 @@ protected void refreshMaximumAllocation(Resource newMaxAlloc) { return null; } + @Override + public List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId) { + SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId); + if (attempt != null) { + return attempt.getAppSchedulingInfo().getAllSchedulingRequests(); + } + return null; + } + @Override public Priority checkAndGetApplicationPriority( Priority priorityRequestedByApp, UserGroupInformation user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8074f069698..d63d2b82fbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -476,6 +476,23 @@ public boolean getAndResetBlacklistChanged() { return ret; } + /** + * Fetch SchedulingRequests. + * @return All pending SchedulingRequests. + */ + public List getAllSchedulingRequests() { + List ret = new ArrayList<>(); + try { + this.readLock.lock(); + schedulerKeyToAppPlacementAllocator.values().stream() + .filter(ap -> ap.getSchedulingRequest() != null) + .forEach(ap -> ret.add(ap.getSchedulingRequest())); + } finally { + this.readLock.unlock(); + } + return ret; + } + public PendingAsk getNextPendingAsk() { try { readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 43d55c4f1dc..0f7a5b5b3ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -363,6 +363,16 @@ public Priority updateApplicationPriority(Priority newPriority, List getPendingResourceRequestsForAttempt( ApplicationAttemptId attemptId); + /** + * Get pending scheduling request for specified application attempt. + * + * @param attemptId the id of the application attempt + * + * @return pending scheduling requests + */ + List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId); + /** * Get cluster max priority. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index df58157efea..088b3dd9c70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -192,4 +192,10 @@ public void initialize(AppSchedulingInfo appSchedulingInfo, this.rmContext = rmContext; this.schedulerRequestKey = schedulerRequestKey; } + + /** + * Get pending SchedulingRequest. + * @return SchedulingRequest + */ + public abstract SchedulingRequest getSchedulingRequest(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index e1239a9db34..f1df3432a86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -425,4 +425,9 @@ public ContainerRequest allocate(SchedulerRequestKey schedulerKey, writeLock.unlock(); } } + + @Override + public SchedulingRequest getSchedulingRequest() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 2b610f2fe76..914f35d7bf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -505,8 +505,8 @@ public void showRequests() { } } - @VisibleForTesting - SchedulingRequest getSchedulingRequest() { + @Override + public SchedulingRequest getSchedulingRequest() { return schedulingRequest; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 43a6ac94b05..b127dc76a8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -90,7 +90,10 @@ private void createResourceRequestsTable(Block html) { .th(".resource", "ResourceName").th(".capacity", "Capability") .th(".containers", "NumContainers") .th(".relaxlocality", "RelaxLocality") - .th(".labelexpression", "NodeLabelExpression").__().__().tbody(); + .th(".labelexpression", "NodeLabelExpression") + .th(".executiontype", "ExecutionType") + .th(".allocationTags", "AllocationTags") + .th(".placementConstraint", "PlacementConstraint").__().__().tbody(); StringBuilder resourceRequestTableData = new StringBuilder("[\n"); for (ResourceRequestInfo resourceRequest : resourceRequests) { @@ -109,6 +112,15 @@ private void createResourceRequestsTable(Block html) { .append("\",\"") .append(resourceRequest.getNodeLabelExpression() == null ? "N/A" : resourceRequest.getNodeLabelExpression()) + .append("\",\"") + .append(resourceRequest.getExecutionTypeRequest() == null ? "N/A" + : resourceRequest.getExecutionTypeRequest().getExecutionType()) + .append("\",\"") + .append(resourceRequest.getAllocationTags() == null ? "N/A" + : resourceRequest.getAllocationTags()) + .append("\",\"") + .append(resourceRequest.getPlacementConstraint() == null ? "N/A" + : resourceRequest.getPlacementConstraint()) .append("\"],\n"); } if (resourceRequestTableData diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9d82bc78c76..63b6fe072d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -230,6 +231,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, resourceRequests.add(new ResourceRequestInfo(req)); } } + + List schedulingRequestsRaw = rm.getRMContext() + .getScheduler().getPendingSchedulingRequestsForAttempt( + attempt.getAppAttemptId()); + if (schedulingRequestsRaw != null) { + for (SchedulingRequest req : schedulingRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java index 030af45cced..d5ee33aad69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java @@ -21,11 +21,13 @@ import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import java.util.Set; /** * Simple class representing a resource request. @@ -50,10 +52,10 @@ @XmlElement(name = "executionTypeRequest") private ExecutionTypeRequestInfo executionTypeRequest; - @XmlElement(name = "executionType") - private String executionType; - @XmlElement(name = "enforceExecutionType") - private boolean enforceExecutionType; + @XmlElement(name = "placementConstraint") + private String placementConstraint; + @XmlElement(name = "allocationTags") + private Set allocationTags; public ResourceRequestInfo() { } @@ -71,6 +73,20 @@ public ResourceRequestInfo(ResourceRequest request) { } } + public ResourceRequestInfo(SchedulingRequest request) { + priority = request.getPriority().getPriority(); + capability = new ResourceInfo(request.getResourceSizing().getResources()); + numContainers = request.getResourceSizing().getNumAllocations(); + if (request.getExecutionType() != null) { + executionTypeRequest = + new ExecutionTypeRequestInfo(request.getExecutionType()); + } + allocationTags = request.getAllocationTags(); + if (request.getPlacementConstraint() != null) { + placementConstraint = request.getPlacementConstraint().toString(); + } + } + public Priority getPriority() { return Priority.newInstance(priority); } @@ -128,4 +144,20 @@ public void setExecutionTypeRequest( public ExecutionTypeRequestInfo getExecutionTypeRequest() { return executionTypeRequest; } + + public String getPlacementConstraint() { + return placementConstraint; + } + + public void setPlacementConstraint(String placementConstraint) { + this.placementConstraint = placementConstraint; + } + + public Set getAllocationTags() { + return allocationTags; + } + + public void setAllocationTags(Set allocationTags) { + this.allocationTags = allocationTags; + } }