From 386a27b68cf724c9f274c76b1f5fba7e8a55ebb7 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Thu, 18 Feb 2021 23:08:16 +0100 Subject: [PATCH] YARN-10636 CS Auto Queue creation should reject submissions with empty path parts Change-Id: I475503ca11a5b7ba2b42f8d10132ca397857bdc8 --- .../scheduler/capacity/CapacityScheduler.java | 15 +++++- ...estCapacitySchedulerAutoQueueCreation.java | 51 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 21d62480fe0..36f831dca94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -962,6 +962,17 @@ private CSQueue getOrCreateQueueFromPlacementContext(ApplicationId if (placementContext == null) { fallbackContext = CSQueueUtils.extractQueuePath(queueName); } + + //we need to make sure there is no empty path parts present + String path = fallbackContext.getFullQueuePath(); + String[] pathParts = path.split("\\."); + for (int i = 0; i < pathParts.length; i++) { + if ("".equals(pathParts[i])) { + LOG.error("Application submitted to invalid path: '{}'", path); + return null; + } + } + if (fallbackContext.hasParentQueue()) { try { return autoCreateLeafQueue(fallbackContext); @@ -1022,8 +1033,8 @@ private void addApplication(ApplicationId applicationId, String queueName, } //Could be a potential auto-created leaf queue - CSQueue queue = getOrCreateQueueFromPlacementContext(applicationId, user, - queueName, placementContext, false); + CSQueue queue = getOrCreateQueueFromPlacementContext( + applicationId, user, queueName, placementContext, false); if (queue == null) { String message; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 4dc0fab1610..57b434f495c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; @@ -89,6 +90,7 @@ .NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -535,6 +537,55 @@ public void testParentQueueUpdateInQueueMappingFailsAfterAutoCreation() } } + + @Test + public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode() + throws Exception { + if (mockRM != null) { + mockRM.stop(); + } + + //We need a special configuration we only need a V2 queue auto creation + //And weight mode, to allow dynamic auto queue creation for root + CapacitySchedulerConfiguration conf = setupSchedulerConfiguration(); + conf.setAutoQueueCreationV2Enabled(ROOT, true); + conf.setCapacity("root.default", "1w"); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + //Just a regular mockRM and CapacityScheduler instance + MockRM newMockRM = new MockRM(conf); + newMockRM.start(); + ((CapacityScheduler) newMockRM.getResourceScheduler()).start(); + + CapacityScheduler newCS = + (CapacityScheduler) newMockRM.getResourceScheduler(); + + try { + //submitting to root..user, this should fail WITHOUT crashing the RM + submitApp(newCS, USER0, "user", "root."); + + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(dispatcher); + newCS.setRMContext(rmContext); + + ApplicationId appId = BuilderUtils.newApplicationId(1, 1); + SchedulerEvent addAppEvent = new AppAddedSchedulerEvent( + appId, "user", "root."); + newCS.handle(addAppEvent); + + RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED, + "error"); + dispatcher.spyOnNextEvent(event, 10000); + } finally { + if (newMockRM != null) { + ((CapacityScheduler) newMockRM.getResourceScheduler()).stop(); + newMockRM.stop(); + } + } + } + + /** * This test case checks if a mapping rule can put an application to an auto * created queue even if an other queue with the same leaf name already -- 2.26.2