From ebf033c2edc9e3870b3f6c4b48df505fa1e0e247 Mon Sep 17 00:00:00 2001 From: Gergely Pollak Date: Tue, 9 Feb 2021 15:02:04 +0100 Subject: [PATCH] YARN-10619 CS Mapping Rule %specified rule catches default submissions Change-Id: I978f5d3cd5458c066753e822979fc3760498bbbc --- .../placement/CSMappingPlacementRule.java | 9 +++- .../placement/TestCSMappingPlacementRule.java | 49 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java index 908498d8c70..43c30f4a0c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java @@ -217,7 +217,13 @@ private VariableContext createVariableContext( VariableContext vctx = new VariableContext(); vctx.put("%user", user); - vctx.put("%specified", asc.getQueue()); + //If the specified matches the default it means NO queue have beeen speified + //as per ClientRMService#submitApplication which sets the queue to default + //when no queue is provided. + //To place queues specificly to default, users must use root.default + if (!asc.getQueue().equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) { + vctx.put("%specified", asc.getQueue()); + } vctx.put("%application", asc.getApplicationName()); vctx.put("%default", "root.default"); try { @@ -379,7 +385,6 @@ public ApplicationPlacementContext getPlacementForApp( asc.getApplicationName(), appQueue, overrideWithQueueMappings); if (appQueue != null && !appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) && - !appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_FULL_NAME) && !overrideWithQueueMappings && !recovery) { LOG.info("Have no jurisdiction over application submission '{}', " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java index 703d517ea7d..69f56ec1e78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java @@ -425,6 +425,55 @@ public void testAllowCreateFlag() throws IOException { } + @Test + public void testSpecified() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%specified", true)) + .setFallbackSkip())); + + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction( + "root.ambiguous.group.tester", true)) + .setFallbackSkip())); + + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.RejectAction()) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext appNoQueue = createApp("app"); + ApplicationSubmissionContext appDefault = createApp("app", "default"); + ApplicationSubmissionContext appRootDefault = + createApp("app", "root.default"); + ApplicationSubmissionContext appBob = + createApp("app", "root.user.bob"); + + assertPlace("App with non specified queue should end up in " + + "'root.ambiguous.group.tester' because no queue was specified and " + + "this is the only rule matching the submission", + engine, appNoQueue, "alice", "root.ambiguous.group.tester"); + + assertPlace("App with specified 'default' should end up in " + + "'root.ambiguous.group.tester' because 'default' is the same as " + + "no queue being specified and this is the only rule matching the " + + "submission ", + engine, appDefault, "alice", "root.ambiguous.group.tester"); + + assertPlace("App with specified root.default should end up in " + + "'root.default' because root.default is specifically provided", + engine, appRootDefault, "alice", "root.default"); + + assertPlace("App with specified queue should end up in the specified " + + "queue 'root.user.bob'", engine, appBob, "alice", "root.user.bob"); + } + private MappingRule createGroupMapping(String group, String queue) { MappingRuleMatcher matcher = MappingRuleMatchers.createUserGroupMatcher(group); MappingRuleAction action = -- 2.26.2