diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/AbstractVisitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/AbstractVisitor.java new file mode 100644 index 0000000..d3485aa --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/AbstractVisitor.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +/** + * Stub class for defining visitors. + */ +class AbstractVisitor extends Visitor { + + @Override + public void visit(Ident node) { + // do nothing + } + + @Override + public void visit(Not node) { + node.expr.accept(this); + } + + @Override + public void visit(And node) { + for (BE child : node.kids) { + child.accept(this); + } + } + + @Override + public void visit(Or node) { + for (BE child : node.kids) { + child.accept(this); + } + } + + @Override + public void visit(Root node) { + node.expr.accept(this); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/BE.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/BE.java new file mode 100644 index 0000000..3fb692e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/BE.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * Boolean expression node type. Boolean expressions are represented as a tree + * of {@link BE.Ident}, {@link BE.And}, {@link BE.Or}, and {@link BE.Not} + * nodes. The {@link Root} node provides a consistent reference for the + * expression as it is mutated by other classes. + * {@see NormalVisitor} + */ +abstract class BE { + + /** + * Visitor API to a BE tree. + */ + public abstract static class Visitor { + public abstract void visit(Ident node); + public abstract void visit(Not node); + public abstract void visit(And node); + public abstract void visit(Or node); + public abstract void visit(Root node); + } + + /** + * Visitor pattern. + * @param v Visitor to pass. + */ + public abstract void accept(Visitor v); + + /** + * Number of leaves below this node. + * @return Leaves across all subexpressions of this node. + */ + public abstract int arity(); + + @Override + public boolean equals(Object other) { + if (!(other instanceof BE)) { + return false; + } + BE o = (BE) other; + EqualVisitor ev = new EqualVisitor(o); + accept(ev); + return ev.result(); + } + + @Override + public int hashCode() { + // hashCode not designed + return getClass().hashCode(); + } + + @Override + public String toString() { + PrintVisitor v = new PrintVisitor(); + accept(v); + return v.toString(); + } + + /** + * Consistent reference for a boolean expression. + */ + public static class Root extends BE { + BE expr; + Root(BE expr) { + this.expr = expr; + } + @Override + public void accept(Visitor v) { + v.visit(this); + } + @Override + public int arity() { + return 1; + } + } + + /** + * Identifier in a boolean expression. + */ + public static class Ident extends BE { + final String name; + Ident(String name) { + this.name = name; + } + @Override + public void accept(Visitor v) { + v.visit(this); + } + @Override + public int arity() { + return 1; + } + @Override + public int hashCode() { + return name.hashCode(); + } + } + + /** + * Negation in a boolean expression. + * Must match {@code !BE} + */ + public static class Not extends BE { + final BE expr; + Not(BE expr) { + this.expr = expr; + } + @Override + public void accept(Visitor v) { + v.visit(this); + } + @Override + public int arity() { + return 1; + } + } + + /** + * Disjunction in a boolean expression. + * Must match {@code BE || BE} + */ + public static class Or extends Operator { + Or(BE... kids) { + this(Arrays.asList(kids)); + } + Or(List kids) { + super(kids); + } + @Override + public void accept(Visitor v) { + v.visit(this); + } + } + + /** + * Conjunction in a boolean expression. + * Must match {@code BE && BE} + */ + public static class And extends Operator { + And(BE... kids) { + this(Arrays.asList(kids)); + } + And(List kids) { + super(kids); + } + @Override + public void accept(Visitor v) { + v.visit(this); + } + } + + /** + * Common subclass for composite boolean expressions. + * {@see And} + * {@see Or} + */ + public abstract static class Operator extends BE { + final List kids; + Operator(List kids) { + this.kids = new ArrayList(kids); + } + @Override + public int arity() { + int ret = 0; + for (BE child : kids) { + ret += child.arity(); + } + return ret; + } + } + + /** + * Build a string representation of the expression. + */ + private static class PrintVisitor extends Visitor { + + private final StringBuilder sb = new StringBuilder(); + + @Override + public void visit(Ident node) { + sb.append(node.name); + } + + @Override + public void visit(Not node) { + sb.append("!("); + node.expr.accept(this); + sb.append(")"); + } + + @Override + public void visit(And node) { + sb.append("("); + Iterator i = node.kids.iterator(); + i.next().accept(this); + while (i.hasNext()) { + sb.append(" && "); + i.next().accept(this); + } + sb.append(")"); + } + + @Override + public void visit(Or node) { + sb.append("("); + Iterator i = node.kids.iterator(); + i.next().accept(this); + while (i.hasNext()) { + sb.append(" || "); + i.next().accept(this); + } + sb.append(")"); + } + + @Override + public void visit(Root node) { + node.expr.accept(this); + } + + @Override + public String toString() { + return sb.toString(); + } + + } + + /** + * Walk hierarchy, verifying exact match of AST and identifiers. + */ + private static class EqualVisitor extends Visitor { + + private BE current; + private boolean result; + + EqualVisitor(BE other) { + result = true; + current = other; + } + + boolean result() { + return result; + } + + @Override + public void visit(Ident node) { + if (!(current instanceof Ident)) { + result = false; + return; + } + Ident other = (Ident) current; + result &= node.name.equals(other.name); + } + + @Override + public void visit(Not node) { + if (!(current instanceof Not)) { + result = false; + return; + } + Not other = (Not) current; + current = other.expr; + node.expr.accept(this); + } + + private void checkOperator(Operator other) { + Iterator i = ((Operator)current).kids.iterator(); + for (BE child : other.kids) { + if (!i.hasNext()) { + result = false; + return; + } + current = i.next(); + child.accept(this); + if (!result) { + return; + } + } + if (i.hasNext()) { + result = false; + } + } + + @Override + public void visit(And node) { + if (!(current instanceof And)) { + result = false; + return; + } + checkOperator(node); + } + + @Override + public void visit(Or node) { + if (!(current instanceof Or)) { + result = false; + return; + } + checkOperator(node); + } + + @Override + public void visit(Root node) { + if (!(current instanceof Root)) { + result = false; + return; + } + Root other = (Root) current; + current = other.expr; + node.expr.accept(this); + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/IntervalSolver.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/IntervalSolver.java new file mode 100644 index 0000000..8b80654 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/IntervalSolver.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Expression matcher based on "Efficiently evaluating complex boolean + * expressions", SIGMOD 2010, Fontoura, et al. + * {@code http://dl.acm.org/citation.cfm?id=1807171} + */ +public class IntervalSolver { + + private static final Logger LOG = + LoggerFactory.getLogger(IntervalSolver.class); + + final int leaves; + final Map> inv; + final Map> notinv; + + /** + * Parse and normalize the given expression to produce a matcher. + * {@link NormalVisitor} + * @param expr A boolean expression matching the grammar accepted by the + * {@link Parser} + * @throws IOException If expression is invalid + */ + public IntervalSolver(String expr) throws IOException { + this(Parser.parse(expr, true)); + } + + /** + * Create a matcher for a parsed, normalized boolean expression. + * {@link NormalVisitor} + * @param expr A parsed and normalized boolean expression. + */ + public IntervalSolver(BE expr) { + this(expr, nLeaves(expr)); + } + + IntervalSolver(BE expr, int leaves) { + IntervalPartitioner ptn = new IntervalPartitioner(leaves); + expr.accept(ptn); + this.inv = Collections.unmodifiableMap(ptn.inv); + this.notinv = Collections.unmodifiableMap(ptn.notinv); + this.leaves = leaves; + LOG.debug("Created {}", expr); + } + + /** + * Return true if the expression evaluates to true when the following + * identifiers are true, and all unmentioned identifiers are false. + * @param ids Identifiers that are true. + * @return true If the expression evaluates to to true when the set + * of identifiers are true, and all unmentioned identifiers are false. + */ + public boolean match(Set ids) { + BitSet b = new BitSet(leaves + 1); + b.set(0); + ArrayList ivs = new ArrayList<>(); + for (String id : ids) { + if (inv.containsKey(id)) { + ivs.addAll(inv.get(id)); + } + } + // add intervals from absent + for (Map.Entry> nid : notinv.entrySet()) { + if (!ids.contains(nid.getKey())) { + ivs.addAll(nid.getValue()); + } + } + + Collections.sort(ivs); + for (Interval i : ivs) { + if (b.get(i.begin - 1)) { + b.set(i.end); + } + } + return b.get(leaves); + } + + static class Interval implements Comparable { + final String ident; // redundant w/ map, but useful for debugging + final int begin; + final int end; + Interval(String ident, int begin, int end) { + this.ident = ident; + this.begin = begin; + this.end = end; + } + @Override + public int hashCode() { + return begin; + } + @Override + public boolean equals(Object other) { + if (!(other instanceof Interval)) { + return false; + } + Interval o = (Interval) other; + return o.begin == begin && + o.end == end && + o.ident.equals(ident); + } + @Override + public int compareTo(Interval o) { + int ret = begin - o.begin; + if (0 == ret) { + ret = ident.compareTo(o.ident); + } + if (0 == ret) { + ret = end - o.end; + } + return ret; + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(begin); + sb.append(" ").append(end); + sb.append(" ").append(ident).append("]"); + return sb.toString(); + } + } + + private static class IntervalPartitioner extends Visitor { + + int begin = 1, end; + int curr = 1; + int left = 0; + final Map> inv = new HashMap<>(); + final Map> notinv = new HashMap<>(); + + IntervalPartitioner(int leaves) { + this.end = leaves; + } + + @Override + public void visit(Ident node) { + ++left; + List l = inv.get(node.name); + if (null == l) { + l = new ArrayList<>(); + inv.put(node.name, l); + } + l.add(new Interval(node.name, begin, end)); + } + + @Override + public void visit(Not node) { + if (!(node.expr instanceof Ident)) { + throw new IllegalStateException(); + } + ++left; + Ident id = (Ident) node.expr; + List l = notinv.get(id.name); + if (null == l) { + l = new ArrayList<>(); + notinv.put(id.name, l); + } + l.add(new Interval(id.name, begin, end)); + } + + @Override + public void visit(And node) { + // first expr + int _end = end; + Iterator i = node.kids.iterator(); + BE first = i.next(); + end = left + first.arity(); + first.accept(this); + curr = end + 1; + // intermediate expr + BE x = null; + while (i.hasNext()) { + x = i.next(); + if (!i.hasNext()) { + break; + } + begin = curr; + end = curr + x.arity() - 1; + x.accept(this); + curr = end + 1; + } + // last expr + begin = curr; + end = _end; + x.accept(this); + } + + @Override + public void visit(Or node) { + for (BE child : node.kids) { + int _begin = begin; + int _end = end; + child.accept(this); + begin = _begin; + end = _end; + } + } + + @Override + public void visit(Root node) { + node.expr.accept(this); + } + + } + + private static int nLeaves(BE expr) { + LeafCount lc = new LeafCount(); + expr.accept(lc); + return lc.leaves(); + } + + static class LeafCount extends AbstractVisitor { + private int nLeaves = -1; + public int leaves() { + if (nLeaves < 0) { + throw new IllegalStateException(); + } + return nLeaves; + } + @Override + public void visit(Root node) { + nLeaves = 0; + super.visit(node); + } + @Override + public void visit(Ident node) { + assert nLeaves >= 0; + ++nLeaves; + } + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NormalVisitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NormalVisitor.java new file mode 100644 index 0000000..2109b75 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NormalVisitor.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +import java.util.ArrayList; +import java.util.ListIterator; + +/** + * Transform the expression to a normal form such that all negations are pushed + * to the leaves and the tree is an alternating sequence of And and Or + * expressions. + */ +class NormalVisitor extends AbstractVisitor { + + private Root norm; + + static BE unNot(BE expr) { + while (expr instanceof Not) { + Not e = (Not) expr; + if (!(e.expr instanceof Not)) { + break; + } + expr = ((Not)e.expr).expr; + } + return expr; + } + + // assume unNot + static BE deM(BE expr) { + if (expr instanceof Not) { + Not e = (Not) expr; + if (e.expr instanceof Ident) { + return expr; + } + Operator o = (Operator) e.expr; + ArrayList k = new ArrayList<>(o.kids.size()); + for (BE child : o.kids) { + k.add(unNot(new Not(child))); + } + if (o instanceof And) { + expr = new Or(k); + } else { + assert o instanceof Or; + expr = new And(k); + } + } + return expr; + } + + @Override + public void visit(Root node) { + norm = new Root(deM(unNot(node.expr))); + super.visit(norm); + node.expr = norm.expr; + } + + @Override + public void visit(Not node) { + if (node.expr instanceof Ident) { + return; + } + throw new IllegalStateException(); + } + + @Override + public void visit(And node) { + for (ListIterator i = node.kids.listIterator(); i.hasNext();) { + BE child = i.next(); + i.set(unNot(deM(child))); + } + super.visit(node); + for (ListIterator i = node.kids.listIterator(); i.hasNext();) { + BE child = i.next(); + if (child instanceof And) { + i.remove(); + for (BE sub : ((And)child).kids) { + i.add(sub); + } + } + } + } + + @Override + public void visit(Or node) { + for (ListIterator i = node.kids.listIterator(); i.hasNext();) { + BE child = i.next(); + i.set(unNot(deM(child))); + } + super.visit(node); + for (ListIterator i = node.kids.listIterator(); i.hasNext();) { + BE child = i.next(); + if (child instanceof Or) { + i.remove(); + for (BE sub : ((Or)child).kids) { + i.add(sub); + } + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/Parser.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/Parser.java new file mode 100644 index 0000000..a56e8ad --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/Parser.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.io.CharArrayReader; +import java.io.IOException; +import java.io.StreamTokenizer; +import java.util.ArrayDeque; +import java.util.Deque; + +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +/** + * Simple shift-reduce parser for boolean expressions. Accepts the following + * grammar: + * {@code ::= "!" } + * {@code | "(" ")"} + * {@code | "&&" } + * {@code | "||" } + * {@code | } + * {@code ::= [A-Za-z]+} + */ +final class Parser { + + enum TType { BE, OPERATOR, LPAREN, RPAREN }; + + private Parser() { + // disable + } + + /** + * Parse the boolean expression but do not normalize it. + * @param expr Boolean expression as a String. + * @return Expression tree as an AST. + */ + public static BE.Root parse(String expr) throws IOException { + return parse(expr, false); + } + + /** + * Parse the boolean expression and normalize the expression for matching. + * {@see NormalVisitor} + * @param expr Boolean expression as a String. + * @return Expression tree as an AST. + */ + public static BE.Root parse(String expr, boolean norm) throws IOException { + Lexer lex = new Lexer(expr); + Deque st = new ArrayDeque(); + Token tok; + while ((tok = lex.next()) != null) { + switch (tok.getType()) { + case LPAREN: + case OPERATOR: + st.push(tok); + break; + case BE: + case RPAREN: + st.push(tok); + reduce(st); + break; + default: + throw new IllegalStateException(); + } + } + if (st.size() > 1) { + reduce(st); + } + if (st.size() == 1) { + BE.Root root = new BE.Root(st.pop().getBE()); + if (norm) { + root.accept(new NormalVisitor()); + } + return root; + } + throw new IOException("Invalid expression: " + expr); + } + + private static void reduce(Deque st) throws IOException { + while (true) { + Token tok = st.pop(); + switch (tok.getType()) { + case BE: + // BE + if (st.isEmpty()) { + st.push(tok); + return; + } + // ... ( BE + if (TType.LPAREN.equals(st.peek().getType())) { + st.push(tok); + return; + } + // ... BE op BE + Token op = st.pop(); + switch (op.getOp()) { + case '!': + st.push(new BEToken(new Not(tok.getBE()))); + break; + case '&': + st.push(new BEToken(new And(st.pop().getBE(), tok.getBE()))); + break; + case '|': + st.push(new BEToken(new Or(st.pop().getBE(), tok.getBE()))); + break; + default: + throw new IOException("Invalid operator " + op); + } + break; + case RPAREN: + // ... ( BE ) + Token betok = st.pop(); + if (!TType.BE.equals(betok.getType())) { + throw new IOException("Unmatched '('"); + } + if (!TType.LPAREN.equals(st.pop().getType())) { + throw new IOException("Unmatched '('"); + } + st.push(betok); + break; + default: + throw new IllegalStateException(); + } + } + } + + private static class Lexer { + private final StreamTokenizer tok; + + Lexer(String s) { + tok = new StreamTokenizer(new CharArrayReader(s.toCharArray())); + tok.parseNumbers(); + tok.ordinaryChar('('); + tok.ordinaryChar(')'); + tok.ordinaryChar('!'); + tok.ordinaryChar('&'); + tok.ordinaryChar('|'); + } + + Token next() throws IOException { + int type = tok.nextToken(); + switch (type) { + case StreamTokenizer.TT_EOF: + case StreamTokenizer.TT_EOL: + return null; + case StreamTokenizer.TT_WORD: + return new BEToken(new BE.Ident(tok.sval)); + default: + switch (type) { + case '(': + return new Token(TType.LPAREN); + case ')': + return new Token(TType.RPAREN); + case '!': + return new OpToken('!'); + case '&': + if ('&' == tok.nextToken()) { + return new OpToken('&'); + } + throw new IOException("Unmatched &"); + case '|': + if ('|' == tok.nextToken()) { + return new OpToken('|'); + } + throw new IOException("Unmatched |"); + default: + throw new IOException("Unexpected: " + (char)type); + } + } + } + } + + private static class Token { + private final TType type; + Token(TType type) { + this.type = type; + } + final TType getType() { + return type; + } + BE getBE() throws IOException { + throw new IOException("Expected node, found " + type); + } + char getOp() throws IOException { + throw new IOException("Expected op, found " + type); + } + @Override + public String toString() { + switch (getType()) { + case LPAREN: return "("; + case RPAREN: return ")"; + default: + throw new IllegalStateException(); + } + } + } + + private static class BEToken extends Token { + private final BE node; + BEToken(BE node) { + super(TType.BE); + this.node = node; + } + @Override + BE getBE() { + return node; + } + @Override + public String toString() { + return node.toString(); + } + } + + private static class OpToken extends Token { + private final char op; + OpToken(char op) { + super(TType.OPERATOR); + this.op = op; + } + @Override + char getOp() { + return op; + } + @Override + public String toString() { + switch (getOp()) { + case '!': return "!"; + case '&': return "&&"; + case '|': return "||"; + default: + throw new IllegalStateException(); + } + } + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestIntervalSolver.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestIntervalSolver.java new file mode 100644 index 0000000..88aca10 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestIntervalSolver.java @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.IntervalSolver.Interval; +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Verify matcher for boolean expressions on both hand-rolled and + * parsed AST. + */ +public class TestIntervalSolver { + + static Interval[] flatten(Map> inv) { + List ret = new ArrayList<>(); + for (Map.Entry> e : inv.entrySet()) { + String key = e.getKey(); + for (Interval x : e.getValue()) { + assertEquals(key, x.ident); + ret.add(x); + } + } + Collections.sort(ret, new Comparator() { + @Override + public int compare(Interval a, Interval b) { + int ret = a.ident.compareTo(b.ident); + if (0 == ret) { + ret = a.begin - b.begin; + } + if (0 == ret) { + ret = a.end - b.end; + } + return ret; + } + }); + return ret.toArray(new Interval[ret.size()]); + } + + @Test + public void testIntervalGen() { + Root root = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("d"), new Ident("e"), new Ident("f")))); + IntervalSolver sv = new IntervalSolver(root); + List soln = Arrays.asList( + new IntervalSolver.Interval("a", 1, 1), + new IntervalSolver.Interval("b", 2, 6), + new IntervalSolver.Interval("c", 1, 6), + new IntervalSolver.Interval("d", 1, 4), + new IntervalSolver.Interval("e", 5, 5), + new IntervalSolver.Interval("f", 6, 6)); + assertEquals(6, sv.leaves); + assertArrayEquals(soln.toArray(), flatten(sv.inv)); + } + + @Test + public void testIntervalGenSamevar() { + Root root = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("a"), new Ident("b"), new Ident("c")))); + IntervalSolver sv = new IntervalSolver(root); + List soln = Arrays.asList( + new IntervalSolver.Interval("a", 1, 1), + new IntervalSolver.Interval("a", 1, 4), + new IntervalSolver.Interval("b", 2, 6), + new IntervalSolver.Interval("b", 5, 5), + new IntervalSolver.Interval("c", 1, 6), + new IntervalSolver.Interval("c", 6, 6)); + assertEquals(6, sv.leaves); + assertArrayEquals(soln.toArray(), flatten(sv.inv)); + } + + @Test + public void testLeafCount() { + Root root = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("d"), new Ident("e"), new Ident("f")))); + IntervalSolver.LeafCount lc = new IntervalSolver.LeafCount(); + root.accept(lc); + assertEquals(6, lc.leaves()); + + root = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And( + new Ident("d"), + new Or( + new Ident("e"), + new And(new Ident("a"), new Ident("c"), new Ident("h")))))); + lc = new IntervalSolver.LeafCount(); + root.accept(lc); + assertEquals(8, lc.leaves()); + } + + @Test + public void testIdentMatch() throws IOException { + Root root = new Root(new Ident("a")); + IntervalSolver sv = new IntervalSolver(root); + chkIdentMatch(sv); + + IntervalSolver psv = new IntervalSolver("a"); + chkIdentMatch(psv); + } + void chkIdentMatch(IntervalSolver sv) { + assertTrue(sv.match(asSet("a"))); + assertFalse(sv.match(asSet("b"))); + assertFalse(sv.match(asSet("c"))); + } + + @Test + public void testSimpleMatch() throws IOException { + Root root = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("d"), new Ident("e"), new Ident("f")))); + IntervalSolver sv = new IntervalSolver(root); + chkSimpleMatch(sv); + + IntervalSolver psv = new IntervalSolver("(a && b) || c || (d && e && f)"); + chkSimpleMatch(psv); + } + void chkSimpleMatch(IntervalSolver sv) { + assertTrue(sv.match(asSet("a", "b"))); + assertTrue(sv.match(asSet("c"))); + assertTrue(sv.match(asSet("d", "e", "f"))); + assertFalse(sv.match(asSet("a"))); + assertFalse(sv.match(asSet("b"))); + assertFalse(sv.match(asSet("d"))); + assertFalse(sv.match(asSet("e"))); + assertFalse(sv.match(asSet("f"))); + assertFalse(sv.match(asSet("d", "e"))); + assertFalse(sv.match(asSet("e", "f"))); + assertFalse(sv.match(asSet("d", "f"))); + } + + @Test + public void testSamevarMatch() throws IOException { + Root root = new Root( + new Or( + new And(new Ident("a"), new Ident("d")), + new Ident("b"), + new And(new Ident("a"), new Ident("c"), new Ident("e")))); + IntervalSolver sv = new IntervalSolver(root); + chkSamevarMatch(sv); + + IntervalSolver psv = new IntervalSolver("(a && d) || b || (a && c && e)"); + chkSamevarMatch(psv); + } + void chkSamevarMatch(IntervalSolver sv) { + assertTrue(sv.match(asSet("a", "d"))); + assertTrue(sv.match(asSet("b"))); + assertTrue(sv.match(asSet("a", "c", "e"))); + assertFalse(sv.match(asSet("a"))); + assertFalse(sv.match(asSet("d"))); + assertFalse(sv.match(asSet("a"))); + assertFalse(sv.match(asSet("c"))); + assertFalse(sv.match(asSet("e"))); + assertFalse(sv.match(asSet("a", "c"))); + assertFalse(sv.match(asSet("a", "e"))); + assertFalse(sv.match(asSet("c", "e"))); + } + + @Test + public void testDeepNested() throws IOException { + IntervalSolver psv = new IntervalSolver( + "a && (b || (c && (d || (e && f && g && h) || i) && j) || k) && l"); + assertTrue(psv.match(asSet("a", "c", "e", "f", "g", "h", "j", "l"))); + } + + @Test + public void testNegatedIdent() throws IOException { + BE root = new Root(new Not(new Ident("a"))); + IntervalSolver sv = new IntervalSolver(root); + chkNegatedIdent(sv); + + IntervalSolver psv = new IntervalSolver("!a"); + chkNegatedIdent(psv); + } + void chkNegatedIdent(IntervalSolver sv) { + assertTrue(sv.match(asSet())); + assertTrue(sv.match(asSet("b", "c"))); + assertFalse(sv.match(asSet("a"))); + assertFalse(sv.match(asSet("a", "b"))); + assertFalse(sv.match(asSet("b", "a"))); + } + + @Test + public void testNegSimple() throws IOException { + BE root = new Root(new And( + new Not(new Ident("a")), + new Ident("b"))); + IntervalSolver sv = new IntervalSolver(root); + chkNegation(sv); + + IntervalSolver psv = new IntervalSolver("!a && b"); + chkNegation(psv); + } + void chkNegation(IntervalSolver sv) { + assertTrue(sv.match(asSet("b"))); + assertFalse(sv.match(asSet())); + } + + @Test + public void testNegationNested() throws IOException { + BE root = new Root(new Or( + new And( + new Not(new Ident("a")), + new Ident("b"), + new Not(new Ident("c"))), + new And( + new Not(new Ident("a")), + new Not(new Ident("b")), + new Ident("c")))); + IntervalSolver sv = new IntervalSolver(root); + chkNegationNested(sv); + + IntervalSolver psv = new IntervalSolver("(!a && b && !c)||(!a && !b && c)"); + chkNegationNested(psv); + } + void chkNegationNested(IntervalSolver sv) { + assertTrue(sv.match(asSet("b"))); + assertTrue(sv.match(asSet("c"))); + assertFalse(sv.match(asSet("a", "b"))); + assertFalse(sv.match(asSet("a", "c"))); + assertFalse(sv.match(asSet("a", "b", "c"))); + assertFalse(sv.match(asSet("b", "c"))); + } + + + private static Set asSet(String... idents) { + HashSet s = new HashSet<>(); + for (String i : idents) { + assertTrue(s.add(i)); + } + return s; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNormalize.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNormalize.java new file mode 100644 index 0000000..c288377 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNormalize.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.io.IOException; + +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.Root; +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NormalVisitor.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Verify the result of {@link NormalVisitor} on expressions. + */ +public class TestNormalize { + + void checkEquals(String expected, String... expr) throws IOException { + BE check = Parser.parse(expected); + check.accept(new NormalVisitor()); + for (String s : expr) { + BE actual = Parser.parse(s); + actual.accept(new NormalVisitor()); + assertEquals(check, actual); + } + } + + @Test + public void testUnNot() throws IOException { + // test internal utility method + BE chk = Parser.parse("!(a && b)"); + BE e1 = Parser.parse("!!!(a && b)"); + assertEquals(((Root)chk).expr, unNot(((Root)e1).expr)); + BE e2 = Parser.parse("!!!!!(a && b)"); + assertEquals(((Root)chk).expr, unNot(((Root)e2).expr)); + } + + @Test + public void testDeM() throws IOException { + // test internal utility method + BE chk = Parser.parse("!a || !b"); + BE e1 = Parser.parse("!(a && b)"); + assertEquals(((Root)chk).expr, deM(unNot(((Root)e1).expr))); + + chk = Parser.parse("!a && !b"); + BE e2 = Parser.parse("!(a || b)"); + assertEquals(((Root)chk).expr, deM(unNot(((Root)e2).expr))); + + // first-level + chk = Parser.parse("!(a && b) || !c"); + BE e3 = Parser.parse("!(a && b && c)"); + assertEquals(((Root)chk).expr, deM(unNot(((Root)e3).expr))); + + } + + @Test + public void testOrCollapse() throws IOException { + checkEquals("a || b || c || d || e || f", + "a || (b || (c || d) || e) || f"); + } + + @Test + public void testAndCollapse() throws IOException { + checkEquals("a && b && c && d && e && f", + "a && (b && (c && d) && e) && f"); + } + + @Test + public void testNotCollapse() throws IOException { + checkEquals("a", "!!a", "!!!!a"); + checkEquals("!a", "!!!a", "!!!!!a"); + } + + @Test + public void testNestedNegation() throws IOException { + checkEquals("!a && !b && !c && !d", + "!a && !(b || c) && !d", + "!(a || b || c || d)"); + checkEquals("!a || !b || !c || !d", + "!a || !(b && c) || !d", + "!(a && b && c && d)"); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestParser.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestParser.java new file mode 100644 index 0000000..0aadf56 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestParser.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; + +import java.io.IOException; + +import static org.apache.hadoop.yarn.server.resourcemanager.nodelabels.BE.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Verify parse tree matches the expression. + */ +public class TestParser { + + @Test + public void testBinaryOperator() throws IOException { + Root a = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Or( + new Ident("c"), + new And( + new Ident("d"), + new And(new Ident("e"), new Ident("f")))))); + Root b = Parser.parse("(a && b) || (c || (d && (e && f)))"); + assertEquals(a, b); + } + + @Test + public void testNaryOperator() throws IOException { + Root a = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("a"), new Ident("b"), new Ident("c")))); + Root b = Parser.parse("(a && b) || c || (a && b && c)"); + b.accept(new NormalVisitor()); + assertEquals(a, b); + + Root c = new Root( + new And(new Ident("a"), new Ident("b"), new Ident("c"), + new Ident("d"), new Ident("e"))); + Root d = Parser.parse("a && b && c && d && e"); + d.accept(new NormalVisitor()); + assertEquals(c, d); + } + + @Test + public void testEquals() { + Root a = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Or( + new Ident("c"), + new And( + new Ident("d"), + new And(new Ident("e"), new Ident("f")))))); + Root b = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Or( + new Ident("c"), + new And( + new Ident("d"), + new And(new Ident("e"), new Ident("f")))))); + assertEquals(a, b); + + Root c = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("d"), new Ident("e"), new Ident("f")))); + Root d = new Root( + new Or( + new And(new Ident("a"), new Ident("b")), + new Ident("c"), + new And(new Ident("d"), new Ident("e"), new Ident("f")))); + assertEquals(c, d); + } + +}