diff --git eclipse-templates/.classpath eclipse-templates/.classpath index 1af41fe..7d682a4 100644 --- eclipse-templates/.classpath +++ eclipse-templates/.classpath @@ -75,6 +75,7 @@ + diff --git ivy/libraries.properties ivy/libraries.properties index d942777..63ae73f 100644 --- ivy/libraries.properties +++ ivy/libraries.properties @@ -40,6 +40,7 @@ commons-lang.version=2.4 commons-logging.version=1.0.4 commons-logging-api.version=1.0.4 commons-pool.version=1.5.4 +commons-lang3.version=3.1 derby.version=10.4.2.0 guava.version=11.0.2 hbase.version=0.94.6.1 diff --git ql/ivy.xml ql/ivy.xml index 276c2ff..d0159d9 100644 --- ql/ivy.xml +++ ql/ivy.xml @@ -66,7 +66,8 @@ + transitive="false"/> + diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index ed78589..798f4a3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -174,10 +174,8 @@ private DAG build(JobConf conf, TezWork work, Path scratchDir, for (BaseWork v: work.getChildren(w)) { assert workToVertex.containsKey(v); Edge e = null; - EdgeType edgeType = EdgeType.SIMPLE_EDGE; - if (work.isBroadCastEdge(w, v)) { - edgeType = EdgeType.BROADCAST_EDGE; - } + + EdgeType edgeType = work.getEdgeProperty(w, v); e = DagUtils.createEdge(wxConf, wx, workToConf.get(v), workToVertex.get(v), edgeType); dag.addEdge(e); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java index 6db0099..5e0ff0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java @@ -25,6 +25,8 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -49,7 +51,8 @@ private final Set leaves = new HashSet(); private final Map> workGraph = new HashMap>(); private final Map> invertedWorkGraph = new HashMap>(); - private final Map> broadcastEdge = new HashMap>(); + private final Map, EdgeType> edgeProperties = + new HashMap, EdgeType>(); /** * getAllWork returns a topologically sorted list of BaseWork @@ -95,7 +98,6 @@ public void add(BaseWork w) { } workGraph.put(w, new LinkedList()); invertedWorkGraph.put(w, new LinkedList()); - broadcastEdge.put(w, new LinkedList()); roots.add(w); leaves.add(w); } @@ -109,13 +111,8 @@ public void connect(BaseWork a, BaseWork b, EdgeType edgeType) { invertedWorkGraph.get(b).add(a); roots.remove(b); leaves.remove(a); - switch (edgeType) { - case BROADCAST_EDGE: - broadcastEdge.get(a).add(b); - break; - default: - break; - } + ImmutablePair workPair = new ImmutablePair(a, b); + edgeProperties.put(workPair, edgeType); } /** @@ -201,11 +198,7 @@ public void remove(BaseWork work) { invertedWorkGraph.remove(work); } - // checks if a and b need a broadcast edge between them - public boolean isBroadCastEdge(BaseWork a, BaseWork b) { - if ((broadcastEdge.get(a).contains(b)) || (broadcastEdge.get(b).contains(a))) { - return true; - } - return false; + public EdgeType getEdgeProperty(BaseWork a, BaseWork b) { + return edgeProperties.get(new ImmutablePair(a,b)); } } diff --git ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java index c4c3ddf..a11d418 100644 --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java @@ -77,6 +77,8 @@ public void testConnect() throws Exception { Assert.assertEquals(work.getParents(w).size(), 0); Assert.assertEquals(work.getChildren(w).size(), 0); } + + Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.SIMPLE_EDGE); } @Test @@ -99,6 +101,8 @@ public void testBroadcastConnect() throws Exception { Assert.assertEquals(work.getParents(w).size(), 0); Assert.assertEquals(work.getChildren(w).size(), 0); } + + Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.BROADCAST_EDGE); } @Test