From 9ae90287652577cdbc6f87f2b7e50c218b5fc051 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 1 Feb 2019 17:10:36 +0530 Subject: [PATCH] YARN-9258 --- .../util/constraint/PlacementConstraintParser.java | 22 ++++++++---------- .../resource/TestPlacementConstraintParser.java | 26 ++++++++++++++++++---- .../distributedshell/PlacementSpec.java | 10 ++++++--- .../src/site/markdown/PlacementConstraints.md.vm | 4 ++-- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java index de9419a..a591432 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java @@ -703,23 +703,19 @@ public static AbstractConstraint parseExpression(String constraintStr) String.valueOf(EXPRESSION_VAL_DELIM), 2); final SourceTags st; final String exprs; - if (splitted.length == 1) { - // source tags not specified - exprs = splitted[0]; - st = SourceTags.emptySourceTags(); - } else if (splitted.length == 2) { - exprs = splitted[1]; + PlacementConstraint constraint; + if (splitted.length == 2 && specStr.length() > 1) { String tagAlloc = splitted[0]; - st = SourceTags.parseFrom(tagAlloc); + st = tagAlloc.length() > 0 ? SourceTags.parseFrom(tagAlloc) : + SourceTags.emptySourceTags(); + exprs = splitted[1]; + constraint = exprs.length() > 0 ? + PlacementConstraintParser.parseExpression(exprs).build() : null; } else { throw new PlacementConstraintParseException( "Unexpected placement constraint expression " + specStr); } - - AbstractConstraint constraint = - PlacementConstraintParser.parseExpression(exprs); - - result.put(st, constraint.build()); + result.put(st, constraint); } // Validation @@ -744,4 +740,4 @@ public static AbstractConstraint parseExpression(String constraintStr) } return result; } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java index 91e4fdb..b9af803 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java @@ -347,6 +347,24 @@ public void testParsePlacementSpec() PlacementConstraint actualPc1, actualPc2; SourceTags tag1, tag2; + // Source Tag without Constraint + result = PlacementConstraintParser + .parsePlacementSpec("foo=2,:bar=3,notin,node,foo"); + Assert.assertEquals(2, result.size()); + Iterator keyIt = result.keySet().iterator(); + tag1 = keyIt.next(); + Assert.assertEquals("foo", tag1.getTag()); + Assert.assertEquals(2, tag1.getNumOfAllocations()); + Iterator valueIt = result.values().iterator(); + actualPc1 = valueIt.next(); + Assert.assertNull(actualPc1); + tag2 = keyIt.next(); + actualPc2 = valueIt.next(); + Assert.assertEquals("bar", tag2.getTag()); + Assert.assertEquals(3, tag2.getNumOfAllocations()); + expectedPc2 = targetNotIn("node", allocationTag("foo")).build(); + Assert.assertEquals(expectedPc2, actualPc2); + // A single anti-affinity constraint result = PlacementConstraintParser .parsePlacementSpec("foo=3,notin,node,foo"); @@ -383,14 +401,14 @@ public void testParsePlacementSpec() result = PlacementConstraintParser .parsePlacementSpec("foo=3,notin,node,foo:bar=2,in,node,foo"); Assert.assertEquals(2, result.size()); - Iterator keyIt = result.keySet().iterator(); + keyIt = result.keySet().iterator(); tag1 = keyIt.next(); Assert.assertEquals("foo", tag1.getTag()); Assert.assertEquals(3, tag1.getNumOfAllocations()); tag2 = keyIt.next(); Assert.assertEquals("bar", tag2.getTag()); Assert.assertEquals(2, tag2.getNumOfAllocations()); - Iterator valueIt = result.values().iterator(); + valueIt = result.values().iterator(); expectedPc1 = targetNotIn("node", allocationTag("foo")).build(); expectedPc2 = targetIn("node", allocationTag("foo")).build(); Assert.assertEquals(expectedPc1, valueIt.next()); @@ -504,7 +522,7 @@ public void testParseNodeAttributeSpec() // A single node attribute constraint w/o source tags result = PlacementConstraintParser - .parsePlacementSpec("rm.yarn.io/foo=true"); + .parsePlacementSpec(",rm.yarn.io/foo=true"); Assert.assertEquals(1, result.size()); target = PlacementTargets.nodeAttribute("rm.yarn.io/foo", "true"); expectedPc1 = targetNodeAttribute("node", NodeAttributeOpCode.EQ, target); @@ -518,7 +536,7 @@ public void testParseNodeAttributeSpec() // then this expression must be single constraint expression. try { PlacementConstraintParser - .parsePlacementSpec("rm.yarn.io/foo=true:xyz=1,notin,node,xyz"); + .parsePlacementSpec(",rm.yarn.io/foo=true:xyz=1,notin,node,xyz"); Assert.fail("Expected a failure!"); } catch (Exception e) { Assert.assertTrue(e instanceof PlacementConstraintParseException); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java index ceaa37d..21b2286 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/PlacementSpec.java @@ -67,7 +67,7 @@ public void setNumContainers(int numContainers) { // PlacementSpec => ""|KeyVal;PlacementSpec // KeyVal => SourceTag=Constraint // SourceTag => String - // Constraint => NumContainers| + // Constraint => NumContainers,| // NumContainers,"in",Scope,TargetTag| // NumContainers,"notin",Scope,TargetTag| // NumContainers,"cardinality",Scope,TargetTag,MinCard,MaxCard @@ -96,8 +96,12 @@ public void setNumContainers(int numContainers) { parsed.entrySet()) { LOG.info("Parsed source tag: {}, number of allocations: {}", entry.getKey().getTag(), entry.getKey().getNumOfAllocations()); - LOG.info("Parsed constraint: {}", entry.getValue() - .getConstraintExpr().getClass().getSimpleName()); + if (entry.getValue() != null) { + LOG.info("Parsed constraint: {}", entry.getValue() + .getConstraintExpr().getClass().getSimpleName()); + } else { + LOG.info("Parsed constraint Empty"); + } pSpecs.put(entry.getKey().getTag(), new PlacementSpec( entry.getKey().getTag(), entry.getKey().getNumOfAllocations(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm index a583493..340a7bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm @@ -66,9 +66,9 @@ where **PlacementSpec** is of the form: ``` PlacementSpec => "" | KeyVal;PlacementSpec -KeyVal => SourceTag=ConstraintExpr +KeyVal => SourceTag=ConstraintExpr | ,Constraint SourceTag => String -ConstraintExpr => NumContainers | NumContainers, Constraint +ConstraintExpr => NumContainers, | NumContainers, Constraint Constraint => SingleConstraint | CompositeConstraint SingleConstraint => "IN",Scope,TargetTag | "NOTIN",Scope,TargetTag | "CARDINALITY",Scope,TargetTag,MinCard,MaxCard CompositeConstraint => AND(ConstraintList) | OR(ConstraintList) -- 2.7.4 (Apple Git-66)