From 5b240878d16187d96dee7eb90dd843e152351949 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 4 Feb 2019 18:22:34 +0530 Subject: [PATCH] YARN-9252 --- .../util/constraint/PlacementConstraintParser.java | 54 ++++++++++------ .../resource/TestPlacementConstraintParser.java | 72 +++++++++++++++++----- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java index de9419a..792aebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java @@ -23,7 +23,9 @@ import org.apache.hadoop.yarn.api.records.NodeAttributeOpCode; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets; import java.util.ArrayList; import java.util.Map; @@ -35,6 +37,7 @@ import java.util.Stack; import java.util.Set; import java.util.TreeSet; +import java.util.HashSet; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,6 +54,7 @@ private static final char KV_SPLIT_DELIM = '='; private static final char BRACKET_START = '('; private static final char BRACKET_END = ')'; + private static final char NAMESPACE_DELIM = '/'; private static final String KV_NE_DELIM = "!="; private static final String IN = "in"; private static final String NOT_IN = "notin"; @@ -104,6 +108,17 @@ int toInt(String name) throws PlacementConstraintParseException { } } + TargetExpression parseNameSpace(String targetTag) + throws PlacementConstraintParseException { + int i = targetTag.lastIndexOf(NAMESPACE_DELIM); + if (i != -1) { + return PlacementTargets.allocationTagWithNamespace( + targetTag.substring(0, i), targetTag.substring(i+1)); + } else { + return PlacementTargets.allocationTag(targetTag); + } + } + String parseScope(String scopeString) throws PlacementConstraintParseException { if (scopeString.equalsIgnoreCase(SCOPE_NODE)) { @@ -392,12 +407,11 @@ public AbstractConstraint parse() + constraintEntities); } - PlacementConstraint.TargetExpression target = null; + TargetExpression target = null; if (!constraintEntities.isEmpty()) { - target = PlacementConstraints.PlacementTargets - .nodeAttribute(attributeName, - constraintEntities - .toArray(new String[constraintEntities.size()])); + target = PlacementTargets.nodeAttribute(attributeName, + constraintEntities + .toArray(new String[constraintEntities.size()])); } placementConstraints = PlacementConstraints @@ -457,23 +471,20 @@ public AbstractConstraint parse() String scope = nextToken(); scope = parseScope(scope); - Set constraintEntities = new TreeSet<>(); + Set targetExpressions = new HashSet<>(); while(hasMoreTokens()) { String tag = nextToken(); - constraintEntities.add(tag); - } - PlacementConstraint.TargetExpression target = null; - if(!constraintEntities.isEmpty()) { - target = PlacementConstraints.PlacementTargets.allocationTag( - constraintEntities - .toArray(new String[constraintEntities.size()])); + TargetExpression t = parseNameSpace(tag); + targetExpressions.add(t); } + TargetExpression[] targetArr = targetExpressions.toArray( + new TargetExpression[targetExpressions.size()]); if (op.equalsIgnoreCase(IN)) { placementConstraints = PlacementConstraints - .targetIn(scope, target); + .targetIn(scope, targetArr); } else { placementConstraints = PlacementConstraints - .targetNotIn(scope, target); + .targetNotIn(scope, targetArr); } } else { throw new PlacementConstraintParseException( @@ -527,13 +538,16 @@ public AbstractConstraint parse() String minCardinalityStr = resetElements.pop(); int min = toInt(minCardinalityStr); - ArrayList targetTags = new ArrayList<>(); + Set targetExpressions = new HashSet<>(); while (!resetElements.empty()) { - targetTags.add(resetElements.pop()); + String tag = resetElements.pop(); + TargetExpression t = parseNameSpace(tag); + targetExpressions.add(t); } + TargetExpression[] targetArr = targetExpressions.toArray( + new TargetExpression[targetExpressions.size()]); - return PlacementConstraints.cardinality(scope, min, max, - targetTags.toArray(new String[targetTags.size()])); + return PlacementConstraints.targetCardinality(scope, min, max, targetArr); } } @@ -744,4 +758,4 @@ public static AbstractConstraint parseExpression(String constraintStr) } return result; } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index 91e4fdb..895acd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -42,7 +42,6 @@ import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and; -import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets; import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn; @@ -110,11 +109,14 @@ public void testTargetExpressionParser() Assert.assertEquals("node", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(0, single.getMaxCardinality()); - Assert.assertEquals(1, single.getTargetExpressions().size()); - TargetExpression exp = - single.getTargetExpressions().iterator().next(); - Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); - Assert.assertEquals(3, exp.getTargetValues().size()); + Assert.assertEquals(3, single.getTargetExpressions().size()); + Set expectedTargetExpressions = Sets.newHashSet( + PlacementTargets.allocationTag("foo"), + PlacementTargets.allocationTag("bar"), + PlacementTargets.allocationTag("exp")); + Assert.assertTrue(Sets.difference(expectedTargetExpressions, + single.getTargetExpressions()).isEmpty()); + verifyConstraintToString(expressionStr, constraint); // Invalid OP @@ -161,13 +163,13 @@ public void testCardinalityConstraintParser() Assert.assertEquals("rack", single.getScope()); Assert.assertEquals(0, single.getMinCardinality()); Assert.assertEquals(1, single.getMaxCardinality()); - Assert.assertEquals(1, single.getTargetExpressions().size()); - exp = single.getTargetExpressions().iterator().next(); - Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString()); - Assert.assertEquals(3, exp.getTargetValues().size()); - Set expectedTags = Sets.newHashSet("foo", "bar", "moo"); - Assert.assertTrue(Sets.difference(expectedTags, exp.getTargetValues()) - .isEmpty()); + Assert.assertEquals(3, single.getTargetExpressions().size()); + Set expectedTargetExpressions = Sets.newHashSet( + PlacementTargets.allocationTag("foo"), + PlacementTargets.allocationTag("bar"), + PlacementTargets.allocationTag("moo")); + Assert.assertTrue(Sets.difference(expectedTargetExpressions, + single.getTargetExpressions()).isEmpty()); verifyConstraintToString(expressionExpr, constraint); // Invalid scope string @@ -376,7 +378,11 @@ public void testParsePlacementSpec() tag1 = result.keySet().iterator().next(); Assert.assertEquals("foo", tag1.getTag()); Assert.assertEquals(10, tag1.getNumOfAllocations()); - expectedPc1 = cardinality("node", 0, 100, "foo", "bar").build(); + TargetExpression[] targetExpressions = new TargetExpression[] { + PlacementTargets.allocationTag("foo"), + PlacementTargets.allocationTag("bar")}; + expectedPc1 = PlacementConstraints.targetCardinality("node", 0, + 100, targetExpressions).build(); Assert.assertEquals(expectedPc1, result.values().iterator().next()); // Two constraint expressions @@ -524,4 +530,42 @@ public void testParseNodeAttributeSpec() Assert.assertTrue(e instanceof PlacementConstraintParseException); } } + + @Test + public void testParseAllocationTagNameSpace() + throws PlacementConstraintParseException { + Map result; + PlacementConstraint.AbstractConstraint expectedPc1, expectedPc2; + PlacementConstraint actualPc1, actualPc2; + + // Constraint with Two Different NameSpaces + result = PlacementConstraintParser + .parsePlacementSpec("foo=2,notin,node,not_self/bar,all/moo"); + Assert.assertEquals(1, result.size()); + Set expectedTargetExpressions = Sets.newHashSet( + PlacementTargets.allocationTagWithNamespace("not_self", "bar"), + PlacementTargets.allocationTagWithNamespace("all", "moo")); + AbstractConstraint constraint = result.values().iterator().next(). + getConstraintExpr(); + Assert.assertTrue(constraint instanceof SingleConstraint); + SingleConstraint single = (SingleConstraint) constraint; + Assert.assertEquals(2, single.getTargetExpressions().size()); + Assert.assertTrue(Sets.difference(expectedTargetExpressions, + single.getTargetExpressions()).isEmpty()); + + // Constraint With Default NameSpace SELF + + result = PlacementConstraintParser + .parsePlacementSpec("foo=2,notin,node,moo"); + Assert.assertEquals(1, result.size()); + TargetExpression expectedTargetExpression = PlacementTargets. + allocationTagWithNamespace("self", "moo"); + constraint = result.values().iterator().next().getConstraintExpr(); + Assert.assertTrue(constraint instanceof SingleConstraint); + single = (SingleConstraint) constraint; + Assert.assertEquals(1, single.getTargetExpressions().size()); + Assert.assertEquals(expectedTargetExpression, + single.getTargetExpressions().iterator().next()); + } + } -- 2.7.4 (Apple Git-66)