From 9816cd2d28aad3cbf990568731da839649c59a9a Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Wed, 9 Dec 2020 17:42:17 +0100 Subject: [PATCH] YARN-10526 RMAppManager CS Placement ignores parent path Change-Id: I17149b068bfde2806062ccb639852354a9e5bcb8 --- .../server/resourcemanager/RMAppManager.java | 16 ++++++++++++- .../scheduler/capacity/CapacityScheduler.java | 23 ++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 13c2ec7de4c..25ff384c742 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -500,11 +500,25 @@ private RMAppImpl createAndPopulateNewRMApp( } } + //In the case of capacity scheduler the queue name only means the name of + // the leaf queue, but since YARN-9879, internal queue references should + // use full path, so we get the queue and parent name from the placement + // context instead of the submissionContext. + String placementQueueName = submissionContext.getQueue(); + if (placementContext != null && scheduler instanceof CapacityScheduler) { + if (placementContext.hasParentQueue()) { + placementQueueName = placementContext.getParentQueue() + "." + + placementContext.getQueue(); + } else { + placementQueueName = placementContext.getQueue(); + } + } + // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, - submissionContext.getQueue(), + placementQueueName, submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), amReqs, placementContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 51df2242be3..b38eee568d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1835,6 +1835,23 @@ CSAssignment allocateContainersToNode( return assignment; } + private String getAddedAppQueueName(AppAddedSchedulerEvent appAddedEvent) { + String ret = appAddedEvent.getQueue(); + ApplicationPlacementContext placementContext = + appAddedEvent.getPlacementContext(); + + if (placementContext != null) { + String leafName = placementContext.getQueue(); + String parentName = placementContext.getParentQueue(); + if (leafName != null) { + ret = placementContext.hasParentQueue() ? + (parentName + "." + leafName) : leafName; + } + } + + return ret; + } + @Override public void handle(SchedulerEvent event) { switch(event.getType()) { @@ -1886,9 +1903,9 @@ public void handle(SchedulerEvent event) { case APP_ADDED: { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; - String queueName = resolveReservationQueueName(appAddedEvent.getQueue(), - appAddedEvent.getApplicationId(), appAddedEvent.getReservationID(), - appAddedEvent.getIsAppRecovering()); + String queueName = resolveReservationQueueName( + getAddedAppQueueName(appAddedEvent), appAddedEvent.getApplicationId(), + appAddedEvent.getReservationID(), appAddedEvent.getIsAppRecovering()); if (queueName != null) { if (!appAddedEvent.getIsAppRecovering()) { addApplication(appAddedEvent.getApplicationId(), queueName, -- 2.26.2