diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java index 4d3658f..6d9739f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.plan; -import java.util.ArrayList; import java.util.LinkedList; import java.util.LinkedHashSet; import java.util.List; @@ -32,14 +31,14 @@ /** * BaseWork. Base class for any "work" that's being done on the cluster. Items like stats - * gathering that are commonly used regarless of the type of work live here. + * gathering that are commonly used regardless of the type of work live here. */ @SuppressWarnings({"serial", "deprecation"}) public abstract class BaseWork extends AbstractOperatorDesc { // dummyOps is a reference to all the HashTableDummy operators in the // plan. These have to be separately initialized when we setup a task. - // Their funtion is mainly as root ops to give the mapjoin the correct + // Their function is mainly as root ops to give the mapjoin the correct // schema info. List dummyOps; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java index 7394588..456b5eb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -34,7 +35,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType; import org.apache.hadoop.mapred.JobConf; -import org.apache.tez.dag.api.EdgeProperty; /** * TezWork. This class encapsulates all the work objects that can be executed @@ -244,7 +244,7 @@ public TezEdgeProperty getEdgeProperty(BaseWork a, BaseWork b) { /* * Dependency is a class used for explain */ - public class Dependency implements Serializable { + public class Dependency implements Serializable, Comparable { public BaseWork w; public EdgeType type; @@ -257,6 +257,15 @@ public String getName() { public String getType() { return type.toString(); } + + @Override + public int compareTo(Dependency o) { + int compare = getName().compareTo(o.getName()); + if (compare == 0) { + compare = getType().compareTo(o.getType()); + } + return compare; + } } @Explain(displayName = "Edges") @@ -271,6 +280,7 @@ public String getType() { dependencies.add(dependency); } if (!dependencies.isEmpty()) { + Collections.sort(dependencies); result.put(entry.getKey().getName(), dependencies); } } diff --git ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out index b407735..73940c6 100644 --- ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out @@ -298,7 +298,7 @@ STAGE PLANS: Tez Edges: Map 2 <- Map 4 (CUSTOM_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -535,7 +535,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 2 (CUSTOM_EDGE), Map 1 (CUSTOM_EDGE) + Map 3 <- Map 1 (CUSTOM_EDGE), Map 2 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1058,7 +1058,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 2 (CUSTOM_EDGE), Map 1 (BROADCAST_EDGE) + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (CUSTOM_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out index eac1669..83af339 100644 --- ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out +++ ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out @@ -92,7 +92,7 @@ STAGE PLANS: Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -187,7 +187,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -304,7 +304,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 diff --git ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out index 5cbfedc..e32b6dd 100644 Binary files ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out and ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out differ diff --git ql/src/test/results/clientpositive/tez/mrr.q.out ql/src/test/results/clientpositive/tez/mrr.q.out index 70c219b..1bb4595 100644 --- ql/src/test/results/clientpositive/tez/mrr.q.out +++ ql/src/test/results/clientpositive/tez/mrr.q.out @@ -1259,7 +1259,7 @@ STAGE PLANS: Reducer 11 <- Reducer 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) diff --git ql/src/test/results/clientpositive/tez/subquery_in.q.out ql/src/test/results/clientpositive/tez/subquery_in.q.out index 7c64f99..2a17cef 100644 --- ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -358,7 +358,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 4 <- Map 5 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -829,7 +829,7 @@ STAGE PLANS: Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/tez/tez_join_tests.q.out ql/src/test/results/clientpositive/tez/tez_join_tests.q.out index 63c8ec0..4e08819 100644 --- ql/src/test/results/clientpositive/tez/tez_join_tests.q.out +++ ql/src/test/results/clientpositive/tez/tez_join_tests.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out index 6f00d74..6d7c83e 100644 --- ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out +++ ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out @@ -14,7 +14,7 @@ STAGE PLANS: Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) + Reducer 4 <- Map 6 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: diff --git ql/src/test/results/clientpositive/tez/tez_union.q.out ql/src/test/results/clientpositive/tez/tez_union.q.out index 4dc687c..4f17445 100644 --- ql/src/test/results/clientpositive/tez/tez_union.q.out +++ ql/src/test/results/clientpositive/tez/tez_union.q.out @@ -16,7 +16,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Union 2 (CONTAINS), Map 3 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: @@ -444,9 +444,9 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS), Map 5 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE) - Map 7 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS), Map 8 (BROADCAST_EDGE) - Map 9 <- Map 8 (BROADCAST_EDGE), Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS), Map 10 (BROADCAST_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 7 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 9 <- Map 1 (BROADCAST_EDGE), Map 10 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -808,8 +808,8 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Union 2 (CONTAINS), Map 3 (BROADCAST_EDGE) - Map 4 <- Union 2 (CONTAINS), Map 5 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 @@ -966,7 +966,7 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 1 <- Union 2 (CONTAINS), Map 3 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) Map 4 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: